aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2015-12-07 12:49:04 +0100
committerCédric Bonhomme <cedric@cedricbonhomme.org>2015-12-07 12:49:04 +0100
commit297ad60fbbc7e7610e2e877d87a1d39b5ffda9ac (patch)
tree33e551c3e5ddae6ce976e12ad048f0e21022764c
parentUpdated README. (diff)
downloadnewspipe-297ad60fbbc7e7610e2e877d87a1d39b5ffda9ac.tar.gz
newspipe-297ad60fbbc7e7610e2e877d87a1d39b5ffda9ac.tar.bz2
newspipe-297ad60fbbc7e7610e2e877d87a1d39b5ffda9ac.zip
Use the PEP 0492 for the legacy crawler.
-rw-r--r--web/crawler.py38
1 files changed, 17 insertions, 21 deletions
diff --git a/web/crawler.py b/web/crawler.py
index 48d00a8b..47ce97ce 100644
--- a/web/crawler.py
+++ b/web/crawler.py
@@ -20,9 +20,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 3.2 $"
+__version__ = "$Revision: 3.3 $"
__date__ = "$Date: 2010/09/02 $"
-__revision__ = "$Date: 2015/04/08 $"
+__revision__ = "$Date: 2015/12/07 $"
__copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "AGPLv3"
@@ -54,8 +54,8 @@ else:
# Handle target environment that doesn't support HTTPS verification
ssl._create_default_https_context = _create_unverified_https_context
-@asyncio.coroutine
-def get(*args, **kwargs):
+
+async def get(*args, **kwargs):
#kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
try:
#logger.info("Fetching the feed: " + args[0])
@@ -66,15 +66,15 @@ def get(*args, **kwargs):
except Exception as e:
raise e
-@asyncio.coroutine
-def parse_feed(user, feed):
+
+async def parse_feed(user, feed):
"""
Fetch a feed.
"""
a_feed = None
- with (yield from sem):
+ with (await sem):
try:
- a_feed = yield from get(feed.link)
+ a_feed = await get(feed.link)
except Exception as e:
feed.last_error = str(e)
finally:
@@ -104,10 +104,9 @@ def parse_feed(user, feed):
return a_feed['entries']
-@asyncio.coroutine
-def insert_database(user, feed):
+async def insert_database(user, feed):
- articles = yield from asyncio.async(parse_feed(user, feed))
+ articles = await parse_feed(user, feed)
if None is articles:
return []
@@ -133,13 +132,14 @@ def insert_database(user, feed):
continue
return new_articles
-@asyncio.coroutine
-def init_process(user, feed):
+
+async def init_process(user, feed):
# Fetch the feed and insert new articles in the database
- articles = yield from asyncio.async(insert_database(user, feed))
+ articles = await insert_database(user, feed)
logger.debug('inserted articles for %s', feed.title)
return articles
+
def retrieve_feed(loop, user, feed_id=None):
"""
Launch the processus.
@@ -155,14 +155,10 @@ def retrieve_feed(loop, user, feed_id=None):
if feeds == []:
return
+
# Launch the process for all the feeds
- tasks = []
- try:
- # Python 3.5 (test)
- tasks = [asyncio.ensure_future(init_process(user, feed))
- for feed in feeds]
- except:
- tasks = [init_process(user, feed) for feed in feeds]
+ tasks = [asyncio.ensure_future(init_process(user, feed)) for feed in feeds]
+
try:
loop.run_until_complete(asyncio.wait(tasks))
except Exception:
bgstack15