adding munin probes

7 files changed, 93 insertions, 4 deletions

diff --git a/conf.py b/conf.py
index 5704ab83..73e83eca 100644
--- a/conf.py
+++ b/conf.py
@@ -28,6 +28,8 @@ DEFAULTS = {"platform_url": "https://pyaggr3g470r.herokuapp.com/",
             "recaptcha_public_key": "",
             "recaptcha_private_key": "",
             "nb_worker": "100",
+            "api_login": "",
+            "api_passwd": "",
             "default_max_error": "3",
             "log_path": "pyaggr3g470r.log",
             "log_level": "info",
@@ -79,6 +81,8 @@ RECAPTCHA_PRIVATE_KEY = config.get('misc',
                                     'recaptcha_private_key')
 LOG_PATH = config.get('misc', 'log_path')
 NB_WORKER = config.getint('misc', 'nb_worker')
+API_LOGIN = config.get('crawler', 'api_login')
+API_PASSWD = config.get('crawler', 'api_passwd')
 
 WHOOSH_ENABLED = True
 
diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample
index 2f30b04a..acd244c8 100644
--- a/conf/conf.cfg-sample
+++ b/conf/conf.cfg-sample
@@ -18,6 +18,8 @@ default_max_error = 6
 http_proxy =
 user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r)
 resolve_article_url = false
+api_login =
+api_passwd =
 [notification]
 notification_email = pyAggr3g470r@no-reply.com
 host = smtp.googlemail.com
diff --git a/manager.py b/manager.py
index 1c038dd6..272dfc2e 100755
--- a/manager.py
+++ b/manager.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from bootstrap import application, db, populate_g
+from bootstrap import application, db, populate_g, conf
 from flask.ext.script import Manager
 from flask.ext.migrate import Migrate, MigrateCommand
 
@@ -27,10 +27,10 @@ def db_create():
         pyaggr3g470r.models.db_create(db)
 
 @manager.command
-def fetch(user, password, limit=100, retreive_all=False):
+def fetch(limit=100, retreive_all=False):
     "Crawl the feeds with the client crawler."
     from pyaggr3g470r.lib.crawler import CrawlerScheduler
-    scheduler = CrawlerScheduler(user, password)
+    scheduler = CrawlerScheduler(conf.API_LOGIN, conf.API_PASSWD)
     scheduler.run(limit=limit, retreive_all=retreive_all)
     scheduler.wait()
 
@@ -66,5 +66,9 @@ def fetch_asyncio(user_id, feed_id):
                 feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
         loop.close()
 
+from scripts.probes import ArticleProbe, FeedProbe
+manager.add_command('probe_articles', ArticleProbe())
+manager.add_command('probe_feeds', FeedProbe())
+
 if __name__ == '__main__':
     manager.run()
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
index 70b9d2dd..21b4b5e7 100644
--- a/pyaggr3g470r/controllers/article.py
+++ b/pyaggr3g470r/controllers/article.py
@@ -33,6 +33,12 @@ class ArticleController(AbstractController):
                               .filter(*self._to_filters(**filters))
                               .group_by(Article.feed_id).all())
 
+    def count_by_user_id(self, **filters):
+        return dict(db.session.query(Article.user_id,
+                                            func.count(Article.id))
+                              .filter(*self._to_filters(**filters))
+                              .group_by(Article.user_id).all())
+
     def create(self, **attrs):
         # handling special denorm for article rights
         assert 'feed_id' in attrs
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
index 530f3fef..ae2cd735 100644
--- a/pyaggr3g470r/views/api/feed.py
+++ b/pyaggr3g470r/views/api/feed.py
@@ -3,6 +3,7 @@
 
 from flask import g
 
+import conf
 from pyaggr3g470r.controllers.feed import (FeedController,
                                            DEFAULT_MAX_ERROR,
                                            DEFAULT_LIMIT,
@@ -54,7 +55,7 @@ class FetchableFeedAPI(PyAggAbstractResource):
         if g.user.refresh_rate:
             args['refresh_rate'] = g.user.refresh_rate
 
-        if args.pop('retreive_all'):
+        if args.pop('retreive_all', False):
             contr = self.wider_controller
         else:
             contr = self.controller
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/__init__.py
diff --git a/scripts/probes.py b/scripts/probes.py
new file mode 100644
index 00000000..bfad4e6e
--- /dev/null
+++ b/scripts/probes.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python3
+import sys
+from datetime import datetime, timedelta
+from flask.ext.script import Command, Option
+
+from pyaggr3g470r.controllers \
+        import UserController, FeedController, ArticleController
+DEFAULT_HEADERS = {'Content-Type': 'application/json', 'User-Agent': 'munin'}
+LATE_AFTER = 60
+FETCH_RATE = 3
+
+
+class AbstractMuninPlugin(Command):
+    urn = None
+
+    def execute(self):
+        raise NotImplementedError()
+
+    def config(self):
+        raise NotImplementedError()
+
+    def get_options(self):
+        if sys.argv[-1] == 'config':
+            return [Option(dest='config', default=sys.argv[-1] == 'config')]
+        return []
+
+    def run(self, config=False):
+        if config:
+            self.config()
+        else:
+            self.execute()
+
+
+class FeedProbe(AbstractMuninPlugin):
+
+    def config(self):
+        print("graph_title PyAgg - Feeds counts")
+        print("graph_vlabel feeds")
+        print("feeds.label Late feeds")
+        print("feeds_total.label Total feeds")
+        print("feeds.warning 15")
+        print("feeds.critical 30")
+        print("graph_category web")
+        print("graph_scale yes")
+
+    def execute(self):
+        delta = datetime.now() - timedelta(minutes=LATE_AFTER + FETCH_RATE + 1)
+
+        print("feeds.value %d" % len(FeedController().list_late(delta)))
+        print("feeds_total.value %d" % FeedController().read().count())
+
+
+class ArticleProbe(AbstractMuninPlugin):
+
+    def config(self):
+        print("graph_title Pyagg - Articles adding rate")
+        print("graph_vlabel Articles per sec")
+        print("articles.label Overall rate")
+        print("articles.type DERIVE")
+        print("articles.min 0")
+        for id_ in sorted(user.id for user in UserController().read()):
+            print("articles_user_%s.label Rate for user %s" % (id_, id_))
+            print("articles_user_%s.type DERIVE" % id_)
+            print("articles_user_%s.min 0" % id_)
+        print("graph_category web")
+        print("graph_scale yes")
+
+    def execute(self):
+        counts = ArticleController().count_by_user_id()
+        print("articles.value %s" % sum(counts.values()))
+        for user, count in counts.items():
+            print("articles_user_%s.value %s" % (user, count))