aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/lib/crawler.py
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-03-01 14:08:02 +0100
committerFrançois Schmidts <francois.schmidts@gmail.com>2015-03-03 22:23:47 +0100
commita4fb151ea53d8054cc8e3fb309395c8fa0e23aaf (patch)
tree6ce8c39978c83d22d2508da67f00c90232819855 /pyaggr3g470r/lib/crawler.py
parentnew crawler with cache control and error handling (diff)
downloadnewspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.tar.gz
newspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.tar.bz2
newspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.zip
fixing/restoring logging level
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
-rw-r--r--pyaggr3g470r/lib/crawler.py7
1 files changed, 4 insertions, 3 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index 6697e4c3..de770934 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -10,6 +10,7 @@ from requests_futures.sessions import FuturesSession
from pyaggr3g470r.lib.utils import default_handler
logger = logging.getLogger(__name__)
+API_ROOT = "api/v2.0/"
def extract_id(entry, keys=[('link', 'link'),
@@ -52,7 +53,7 @@ class AbstractCrawler:
if data is None:
data = {}
method = getattr(self.session, method)
- return method("%sapi/v1.0/%s" % (self.url, urn),
+ return method("%s%s%s" % (self.url, API_ROOT, urn),
auth=self.auth, data=json.dumps(data,
default=default_handler),
headers={'Content-Type': 'application/json'})
@@ -193,7 +194,7 @@ class CrawlerScheduler(AbstractCrawler):
headers=self.prepare_headers(feed))
future.add_done_callback(FeedCrawler(feed, self.auth).callback)
- def run(self):
+ def run(self, **kwargs):
logger.debug('retreving fetchable feed')
- future = self.query_pyagg('get', 'feeds/fetchable')
+ future = self.query_pyagg('get', 'feeds/fetchable', kwargs)
future.add_done_callback(self.callback)
bgstack15