aboutsummaryrefslogtreecommitdiff
path: root/web/lib/article_utils.py
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-12-12 21:14:28 +0100
committerFrançois Schmidts <francois.schmidts@gmail.com>2015-12-17 09:42:56 +0100
commitb35e9773198ef2d8b37c4ca223f08147db47de0b (patch)
treeba4b1b171b3c1ab9414a96ad264c47b0f9d1246b /web/lib/article_utils.py
parentUpdated link to Heroku deploy button on the About page. (diff)
downloadnewspipe-b35e9773198ef2d8b37c4ca223f08147db47de0b.tar.gz
newspipe-b35e9773198ef2d8b37c4ca223f08147db47de0b.tar.bz2
newspipe-b35e9773198ef2d8b37c4ca223f08147db47de0b.zip
moving the root of source code from / to /src/
Diffstat (limited to 'web/lib/article_utils.py')
-rw-r--r--web/lib/article_utils.py74
1 files changed, 0 insertions, 74 deletions
diff --git a/web/lib/article_utils.py b/web/lib/article_utils.py
deleted file mode 100644
index 02ca2cd1..00000000
--- a/web/lib/article_utils.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import logging
-import requests
-import dateutil.parser
-from datetime import datetime
-
-import conf
-from web.lib.utils import to_hash
-
-logger = logging.getLogger(__name__)
-
-
-def extract_id(entry, keys=[('link', 'link'), ('published', 'date'),
- ('updated', 'date')], force_id=False):
- """For a given entry will return a dict that allows to identify it. The
- dict will be constructed on the uid of the entry. if that identifier is
- absent, the dict will be constructed upon the values of "keys".
- """
- entry_id = entry.get('entry_id') or entry.get('id')
- if entry_id:
- return {'entry_id': entry_id}
- if not entry_id and force_id:
- return to_hash("".join(entry[entry_key] for _, entry_key in keys
- if entry_key in entry).encode('utf8'))
- else:
- ids = {}
- for entry_key, pyagg_key in keys:
- if entry_key in entry and pyagg_key not in ids:
- ids[pyagg_key] = entry[entry_key]
- if 'date' in pyagg_key:
- ids[pyagg_key] = dateutil.parser.parse(ids[pyagg_key])\
- .isoformat()
- return ids
-
-
-def construct_article(entry, feed):
- if hasattr(feed, 'dump'): # this way can be a sqlalchemy obj or a dict
- feed = feed.dump()
- "Safe method to transorm a feedparser entry into an article"
- now = datetime.now()
- date = None
- for date_key in ('published', 'updated'):
- if entry.get(date_key):
- try:
- date = dateutil.parser.parse(entry[date_key])
- except Exception:
- pass
- else:
- break
- content = ''
- if entry.get('content'):
- content = entry['content'][0]['value']
- elif entry.get('summary'):
- content = entry['summary']
-
- article_link = entry.get('link')
- if conf.RESOLVE_ARTICLE_URL and article_link:
- try:
- # resolves URL behind proxies
- # (like feedproxy.google.com)
- response = requests.get(article_link, verify=False, timeout=5.0)
- article_link = response.url
- except Exception as error:
- logger.warning("Unable to get the real URL of %s. Error: %s",
- article_link, error)
-
- return {'feed_id': feed['id'],
- 'user_id': feed['user_id'],
- 'entry_id': extract_id(entry).get('entry_id', None),
- 'link': entry.get('link', feed['site_link']),
- 'title': entry.get('title', 'No title'),
- 'readed': False, 'like': False,
- 'content': content,
- 'retrieved_date': now.isoformat(),
- 'date': (date or now).isoformat()}
bgstack15