diff options
| author | François Schmidts <francois.schmidts@gmail.com> | 2015-03-01 14:08:02 +0100 |
|---|---|---|
| committer | François Schmidts <francois.schmidts@gmail.com> | 2015-03-03 22:23:47 +0100 |
| commit | a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf (patch) | |
| tree | 6ce8c39978c83d22d2508da67f00c90232819855 /pyaggr3g470r/lib | |
| parent | new crawler with cache control and error handling (diff) | |
| download | newspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.tar.gz newspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.tar.bz2 newspipe-a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf.zip | |
fixing/restoring logging level
Diffstat (limited to 'pyaggr3g470r/lib')
| -rwxr-xr-x | pyaggr3g470r/lib/client.py | 16 | ||||
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/lib/exceptions.py | 13 |
3 files changed, 4 insertions, 32 deletions
diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py deleted file mode 100755 index 6b2fc9ae..00000000 --- a/pyaggr3g470r/lib/client.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import json -import requests -import conf - - -def get_client(email, password): - client = requests.session() - client.get(conf.PLATFORM_URL + 'api/csrf', verify=False, - data=json.dumps({'email': email, - 'password': password})) - return client - - -def get_articles(client): - return client.get(conf.PLATFORM_URL + 'api/v1.0/articles/').json diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 6697e4c3..de770934 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -10,6 +10,7 @@ from requests_futures.sessions import FuturesSession from pyaggr3g470r.lib.utils import default_handler logger = logging.getLogger(__name__) +API_ROOT = "api/v2.0/" def extract_id(entry, keys=[('link', 'link'), @@ -52,7 +53,7 @@ class AbstractCrawler: if data is None: data = {} method = getattr(self.session, method) - return method("%sapi/v1.0/%s" % (self.url, urn), + return method("%s%s%s" % (self.url, API_ROOT, urn), auth=self.auth, data=json.dumps(data, default=default_handler), headers={'Content-Type': 'application/json'}) @@ -193,7 +194,7 @@ class CrawlerScheduler(AbstractCrawler): headers=self.prepare_headers(feed)) future.add_done_callback(FeedCrawler(feed, self.auth).callback) - def run(self): + def run(self, **kwargs): logger.debug('retreving fetchable feed') - future = self.query_pyagg('get', 'feeds/fetchable') + future = self.query_pyagg('get', 'feeds/fetchable', kwargs) future.add_done_callback(self.callback) diff --git a/pyaggr3g470r/lib/exceptions.py b/pyaggr3g470r/lib/exceptions.py deleted file mode 100644 index 30c71a5c..00000000 --- a/pyaggr3g470r/lib/exceptions.py +++ /dev/null @@ -1,13 +0,0 @@ -class PyAggError(Exception): - status_code = None - default_message = '' - - -class Forbidden(PyAggError): - status_code = 403 - default_message = 'You do not have the rights to access that resource' - - -class NotFound(PyAggError): - status_code = 404 - default_message = 'Resource was not found' |