diff options
| author | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2014-01-19 21:59:24 +0100 |
|---|---|---|
| committer | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2014-01-19 21:59:24 +0100 |
| commit | a9df02d807da6622bd15d9c10d6a11866335eeeb (patch) | |
| tree | e60266514e68736b69bd6e6c408ee51eb097cc35 | |
| parent | Only get the URL with requests if the string 'feedproxy' is found in the netl... (diff) | |
| download | newspipe-a9df02d807da6622bd15d9c10d6a11866335eeeb.tar.gz newspipe-a9df02d807da6622bd15d9c10d6a11866335eeeb.tar.bz2 newspipe-a9df02d807da6622bd15d9c10d6a11866335eeeb.zip | |
The MongoEngine object is saved after all threads are done.
| -rw-r--r-- | pyaggr3g470r/feedgetter.py | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index da28663c..cc9f8aa4 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -96,6 +96,8 @@ class FeedGetter(object): for th in list_of_threads: th.join() + self.user.save() + def process(self, feed): """ Retrieves articles form the feed and add them to the database. @@ -111,10 +113,11 @@ class FeedGetter(object): if "feedproxy" in urlparse(nice_url).netloc: try: # resolves URL behind proxies (like feedproxy.google.com) - r = requests.get(article.link, timeout=10.0, proxies=self.proxies) + r = requests.get(article.link, timeout=5.0, proxies=self.proxies) nice_url = r.url.encode("utf-8") except Timeout: pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,)) + print "Time out" continue except Exception as e: pyaggr3g470r_log.warning("Unable to get the real URL of %s. Error: %s" % (article.link, str(e))) @@ -187,7 +190,7 @@ class FeedGetter(object): # add the articles to the list of articles for the current feed feed.articles.extend(articles) feed.articles = sorted(feed.articles, key=lambda t: t.date, reverse=True) - self.user.save() + return True if __name__ == "__main__": |