aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/crawler/classic_crawler.py2
-rw-r--r--src/web/lib/article_utils.py16
2 files changed, 11 insertions, 7 deletions
diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py
index cd395d0f..8b952940 100644
--- a/src/crawler/classic_crawler.py
+++ b/src/crawler/classic_crawler.py
@@ -148,6 +148,8 @@ async def insert_database(user, feed):
existing_article.dump())
continue
article = construct_article(article, feed)
+ print("test.............................................")
+ print(article)
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
diff --git a/src/web/lib/article_utils.py b/src/web/lib/article_utils.py
index 46bb9461..b5b9f246 100644
--- a/src/web/lib/article_utils.py
+++ b/src/web/lib/article_utils.py
@@ -36,21 +36,23 @@ def construct_article(entry, feed):
feed = feed.dump()
"Safe method to transorm a feedparser entry into an article"
now = datetime.now()
- date, updated_date = None, None
- for date_key in ('published', 'updated'):
+ date = None
+ for date_key in ('published', 'created', 'date'):
if entry.get(date_key):
try:
- date = dateutil.parser.parse(entry[date_key])
+ date = dateutil.parser.parse(entry[date_key])\
+ .astimezone(timezone.utc)
except Exception:
pass
else:
break
+
+ updated_date = None
try:
updated_date = dateutil.parser.parse(entry['updated'])
except Exception:
pass
content = get_article_content(entry)
-
article_link = entry.get('link')
return {'feed_id': feed['id'],
@@ -60,9 +62,9 @@ def construct_article(entry, feed):
'title': entry.get('title', 'No title'),
'readed': False, 'like': False,
'content': content,
- 'retrieved_date': now.isoformat(),
- 'date': (date or now).isoformat(),
- 'updated_date': (updated_date or date or now).isoformat()}
+ 'retrieved_date': now,
+ 'date': date or now,
+ 'updated_date': updated_date or date or now}
def get_article_content(entry):
content = ''
bgstack15