diff options
| author | François Schmidts <francois.schmidts@gmail.com> | 2015-03-10 09:25:10 +0100 |
|---|---|---|
| committer | François Schmidts <francois.schmidts@gmail.com> | 2015-03-10 09:25:10 +0100 |
| commit | 0cc0e87d3f3bafba6a22c883cdf24e9962fafe37 (patch) | |
| tree | 2a673a9809fad4c34733355474c766f7c0de0786 /pyaggr3g470r/lib | |
| parent | better crawling crontrol (diff) | |
| download | newspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.tar.gz newspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.tar.bz2 newspipe-0cc0e87d3f3bafba6a22c883cdf24e9962fafe37.zip | |
failover for bad counter manipulation
Diffstat (limited to 'pyaggr3g470r/lib')
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 8e61b7cf..9df37993 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -107,11 +107,17 @@ class AbstractCrawler: 'User-Agent': 'pyaggr3g470r'}) @classmethod - def wait(cls): + def wait(cls, max_wait=600): "See count_on_me, that method will just wait for the counter to be 0" time.sleep(1) + second_waited = 1 while cls.__counter__: + if second_waited > max_wait: + logger.warn('Exiting after %d seconds, counter at %d', + max_wait, cls.__counter__) + break time.sleep(1) + second_waited += 1 class PyAggUpdater(AbstractCrawler): |