From 7d9bc6621eea035034cd56d20de9333f09387940 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Tue, 21 May 2019 23:49:03 +0200 Subject: Improved crawler. --- src/manager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src/manager.py') diff --git a/src/manager.py b/src/manager.py index 795b3974..244ab77f 100755 --- a/src/manager.py +++ b/src/manager.py @@ -65,16 +65,19 @@ def fetch_asyncio(user_id=None, feed_id=None): feed_id = int(feed_id) except: feed_id = None + + + loop = asyncio.get_event_loop() + queue = asyncio.Queue(maxsize=2, loop=loop) + + producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) + consumer_coro = default_crawler.insert_articles(queue, 1) logger.info('Starting crawler.') - start = datetime.now() - loop = asyncio.get_event_loop() - for user in users: - default_crawler.retrieve_feed(loop, user, feed_id) - loop.close() + loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) end = datetime.now() - + loop.close() logger.info('Crawler finished in {} seconds.' \ .format((end - start).seconds)) -- cgit