aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-07-31 13:15:16 +0200
committerFrançois Schmidts <francois.schmidts@gmail.com>2015-07-31 13:15:16 +0200
commit3b3f188e1558987a8a14dc303912e208f77c7b60 (patch)
treeaa75bab5040e0c15db0d3c9942695779e2c696c7
parentThe numver ov values of the splited string is variable (sometimes the charset... (diff)
downloadnewspipe-3b3f188e1558987a8a14dc303912e208f77c7b60.tar.gz
newspipe-3b3f188e1558987a8a14dc303912e208f77c7b60.tar.bz2
newspipe-3b3f188e1558987a8a14dc303912e208f77c7b60.zip
adding munin probes
-rw-r--r--conf.py4
-rw-r--r--conf/conf.cfg-sample2
-rwxr-xr-xmanager.py10
-rw-r--r--pyaggr3g470r/controllers/article.py6
-rw-r--r--pyaggr3g470r/views/api/feed.py3
-rw-r--r--scripts/__init__.py0
-rw-r--r--scripts/probes.py72
7 files changed, 93 insertions, 4 deletions
diff --git a/conf.py b/conf.py
index 5704ab83..73e83eca 100644
--- a/conf.py
+++ b/conf.py
@@ -28,6 +28,8 @@ DEFAULTS = {"platform_url": "https://pyaggr3g470r.herokuapp.com/",
"recaptcha_public_key": "",
"recaptcha_private_key": "",
"nb_worker": "100",
+ "api_login": "",
+ "api_passwd": "",
"default_max_error": "3",
"log_path": "pyaggr3g470r.log",
"log_level": "info",
@@ -79,6 +81,8 @@ RECAPTCHA_PRIVATE_KEY = config.get('misc',
'recaptcha_private_key')
LOG_PATH = config.get('misc', 'log_path')
NB_WORKER = config.getint('misc', 'nb_worker')
+API_LOGIN = config.get('crawler', 'api_login')
+API_PASSWD = config.get('crawler', 'api_passwd')
WHOOSH_ENABLED = True
diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample
index 2f30b04a..acd244c8 100644
--- a/conf/conf.cfg-sample
+++ b/conf/conf.cfg-sample
@@ -18,6 +18,8 @@ default_max_error = 6
http_proxy =
user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r)
resolve_article_url = false
+api_login =
+api_passwd =
[notification]
notification_email = pyAggr3g470r@no-reply.com
host = smtp.googlemail.com
diff --git a/manager.py b/manager.py
index 1c038dd6..272dfc2e 100755
--- a/manager.py
+++ b/manager.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from bootstrap import application, db, populate_g
+from bootstrap import application, db, populate_g, conf
from flask.ext.script import Manager
from flask.ext.migrate import Migrate, MigrateCommand
@@ -27,10 +27,10 @@ def db_create():
pyaggr3g470r.models.db_create(db)
@manager.command
-def fetch(user, password, limit=100, retreive_all=False):
+def fetch(limit=100, retreive_all=False):
"Crawl the feeds with the client crawler."
from pyaggr3g470r.lib.crawler import CrawlerScheduler
- scheduler = CrawlerScheduler(user, password)
+ scheduler = CrawlerScheduler(conf.API_LOGIN, conf.API_PASSWD)
scheduler.run(limit=limit, retreive_all=retreive_all)
scheduler.wait()
@@ -66,5 +66,9 @@ def fetch_asyncio(user_id, feed_id):
feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
loop.close()
+from scripts.probes import ArticleProbe, FeedProbe
+manager.add_command('probe_articles', ArticleProbe())
+manager.add_command('probe_feeds', FeedProbe())
+
if __name__ == '__main__':
manager.run()
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
index 70b9d2dd..21b4b5e7 100644
--- a/pyaggr3g470r/controllers/article.py
+++ b/pyaggr3g470r/controllers/article.py
@@ -33,6 +33,12 @@ class ArticleController(AbstractController):
.filter(*self._to_filters(**filters))
.group_by(Article.feed_id).all())
+ def count_by_user_id(self, **filters):
+ return dict(db.session.query(Article.user_id,
+ func.count(Article.id))
+ .filter(*self._to_filters(**filters))
+ .group_by(Article.user_id).all())
+
def create(self, **attrs):
# handling special denorm for article rights
assert 'feed_id' in attrs
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
index 530f3fef..ae2cd735 100644
--- a/pyaggr3g470r/views/api/feed.py
+++ b/pyaggr3g470r/views/api/feed.py
@@ -3,6 +3,7 @@
from flask import g
+import conf
from pyaggr3g470r.controllers.feed import (FeedController,
DEFAULT_MAX_ERROR,
DEFAULT_LIMIT,
@@ -54,7 +55,7 @@ class FetchableFeedAPI(PyAggAbstractResource):
if g.user.refresh_rate:
args['refresh_rate'] = g.user.refresh_rate
- if args.pop('retreive_all'):
+ if args.pop('retreive_all', False):
contr = self.wider_controller
else:
contr = self.controller
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/__init__.py
diff --git a/scripts/probes.py b/scripts/probes.py
new file mode 100644
index 00000000..bfad4e6e
--- /dev/null
+++ b/scripts/probes.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python3
+import sys
+from datetime import datetime, timedelta
+from flask.ext.script import Command, Option
+
+from pyaggr3g470r.controllers \
+ import UserController, FeedController, ArticleController
+DEFAULT_HEADERS = {'Content-Type': 'application/json', 'User-Agent': 'munin'}
+LATE_AFTER = 60
+FETCH_RATE = 3
+
+
+class AbstractMuninPlugin(Command):
+ urn = None
+
+ def execute(self):
+ raise NotImplementedError()
+
+ def config(self):
+ raise NotImplementedError()
+
+ def get_options(self):
+ if sys.argv[-1] == 'config':
+ return [Option(dest='config', default=sys.argv[-1] == 'config')]
+ return []
+
+ def run(self, config=False):
+ if config:
+ self.config()
+ else:
+ self.execute()
+
+
+class FeedProbe(AbstractMuninPlugin):
+
+ def config(self):
+ print("graph_title PyAgg - Feeds counts")
+ print("graph_vlabel feeds")
+ print("feeds.label Late feeds")
+ print("feeds_total.label Total feeds")
+ print("feeds.warning 15")
+ print("feeds.critical 30")
+ print("graph_category web")
+ print("graph_scale yes")
+
+ def execute(self):
+ delta = datetime.now() - timedelta(minutes=LATE_AFTER + FETCH_RATE + 1)
+
+ print("feeds.value %d" % len(FeedController().list_late(delta)))
+ print("feeds_total.value %d" % FeedController().read().count())
+
+
+class ArticleProbe(AbstractMuninPlugin):
+
+ def config(self):
+ print("graph_title Pyagg - Articles adding rate")
+ print("graph_vlabel Articles per sec")
+ print("articles.label Overall rate")
+ print("articles.type DERIVE")
+ print("articles.min 0")
+ for id_ in sorted(user.id for user in UserController().read()):
+ print("articles_user_%s.label Rate for user %s" % (id_, id_))
+ print("articles_user_%s.type DERIVE" % id_)
+ print("articles_user_%s.min 0" % id_)
+ print("graph_category web")
+ print("graph_scale yes")
+
+ def execute(self):
+ counts = ArticleController().count_by_user_id()
+ print("articles.value %s" % sum(counts.values()))
+ for user, count in counts.items():
+ print("articles_user_%s.value %s" % (user, count))
bgstack15