aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-03-10 09:25:10 +0100
committerFrançois Schmidts <francois.schmidts@gmail.com>2015-03-10 09:25:10 +0100
commit0cc0e87d3f3bafba6a22c883cdf24e9962fafe37 (patch)
tree2a673a9809fad4c34733355474c766f7c0de0786
parentbetter crawling crontrol (diff)
downloadnewspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.tar.gz
newspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.tar.bz2
newspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.zip
failover for bad counter manipulation
-rw-r--r--pyaggr3g470r/lib/crawler.py8
1 files changed, 7 insertions, 1 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index 8e61b7cf..9df37993 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -107,11 +107,17 @@ class AbstractCrawler:
'User-Agent': 'pyaggr3g470r'})
@classmethod
- def wait(cls):
+ def wait(cls, max_wait=600):
"See count_on_me, that method will just wait for the counter to be 0"
time.sleep(1)
+ second_waited = 1
while cls.__counter__:
+ if second_waited > max_wait:
+ logger.warn('Exiting after %d seconds, counter at %d',
+ max_wait, cls.__counter__)
+ break
time.sleep(1)
+ second_waited += 1
class PyAggUpdater(AbstractCrawler):
bgstack15