aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/crawler.py
diff options
context:
space:
mode:
Diffstat (limited to 'pyaggr3g470r/crawler.py')
-rw-r--r--pyaggr3g470r/crawler.py10
1 files changed, 6 insertions, 4 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index 7815a631..1bc224fb 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -40,6 +40,12 @@ from pyaggr3g470r.models import User, Article
logger = logging.getLogger(__name__)
+#
+# asyncio examples:
+# -http://compiletoi.net/fast-scraping-in-python-with-asyncio.html
+# - https://gist.github.com/kunev/f83146d407c81a2d64a6
+#
+
@asyncio.coroutine
def fetch(user, feed):
"""
@@ -167,8 +173,6 @@ def insert_database(user, feed):
#db.session.close()
return new_articles
-
-
@asyncio.coroutine
def done(feed):
print('done {}'.format(feed.title))
@@ -178,8 +182,6 @@ def process_data(user, feed):
data = yield from asyncio.async(insert_database(user, feed))
print('inserted articles for {}'.format(feed.title))
-
-
def retrieve_feed(user, feed_id=None):
"""
Launch the processus.
bgstack15