diff options
Diffstat (limited to 'web')
| -rw-r--r-- | web/crawler.py | 38 |
1 files changed, 17 insertions, 21 deletions
diff --git a/web/crawler.py b/web/crawler.py index 48d00a8b..47ce97ce 100644 --- a/web/crawler.py +++ b/web/crawler.py @@ -20,9 +20,9 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. __author__ = "Cedric Bonhomme" -__version__ = "$Revision: 3.2 $" +__version__ = "$Revision: 3.3 $" __date__ = "$Date: 2010/09/02 $" -__revision__ = "$Date: 2015/04/08 $" +__revision__ = "$Date: 2015/12/07 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "AGPLv3" @@ -54,8 +54,8 @@ else: # Handle target environment that doesn't support HTTPS verification ssl._create_default_https_context = _create_unverified_https_context -@asyncio.coroutine -def get(*args, **kwargs): + +async def get(*args, **kwargs): #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False) try: #logger.info("Fetching the feed: " + args[0]) @@ -66,15 +66,15 @@ def get(*args, **kwargs): except Exception as e: raise e -@asyncio.coroutine -def parse_feed(user, feed): + +async def parse_feed(user, feed): """ Fetch a feed. """ a_feed = None - with (yield from sem): + with (await sem): try: - a_feed = yield from get(feed.link) + a_feed = await get(feed.link) except Exception as e: feed.last_error = str(e) finally: @@ -104,10 +104,9 @@ def parse_feed(user, feed): return a_feed['entries'] -@asyncio.coroutine -def insert_database(user, feed): +async def insert_database(user, feed): - articles = yield from asyncio.async(parse_feed(user, feed)) + articles = await parse_feed(user, feed) if None is articles: return [] @@ -133,13 +132,14 @@ def insert_database(user, feed): continue return new_articles -@asyncio.coroutine -def init_process(user, feed): + +async def init_process(user, feed): # Fetch the feed and insert new articles in the database - articles = yield from asyncio.async(insert_database(user, feed)) + articles = await insert_database(user, feed) logger.debug('inserted articles for %s', feed.title) return articles + def retrieve_feed(loop, user, feed_id=None): """ Launch the processus. @@ -155,14 +155,10 @@ def retrieve_feed(loop, user, feed_id=None): if feeds == []: return + # Launch the process for all the feeds - tasks = [] - try: - # Python 3.5 (test) - tasks = [asyncio.ensure_future(init_process(user, feed)) - for feed in feeds] - except: - tasks = [init_process(user, feed) for feed in feeds] + tasks = [asyncio.ensure_future(init_process(user, feed)) for feed in feeds] + try: loop.run_until_complete(asyncio.wait(tasks)) except Exception: |