From eb439384ebfe13bc95d1065fb8fb507453533efd Mon Sep 17 00:00:00 2001 From: cedricbonhomme Date: Sat, 3 Mar 2012 18:46:28 +0100 Subject: Main page almost working. --- feedgetter.py | 17 ++++- mongodb.py | 83 +++++++++++++++++----- pyAggr3g470r.py | 211 +++++++++++++++++++++++++++++++------------------------- utils.py | 6 +- 4 files changed, 200 insertions(+), 117 deletions(-) diff --git a/feedgetter.py b/feedgetter.py index cc4fb5ca..74b7b3a0 100755 --- a/feedgetter.py +++ b/feedgetter.py @@ -30,6 +30,7 @@ import traceback import sqlite3 import threading import feedparser +import hashlib from BeautifulSoup import BeautifulSoup from datetime import datetime @@ -118,7 +119,12 @@ class FeedGetter(object): except: feed_image = "/img/feed-icon-28x28.png" - collection_dic = {"collection_id": feed_link,\ + sha1_hash = hashlib.sha1() + sha1_hash.update(feed_link.encode('utf-8')) + feed_id = sha1_hash.hexdigest() + + collection_dic = {"feed_id": feed_id, \ + "type": 0, \ "feed_image": feed_image, \ "feed_title": utils.clear_string(a_feed.feed.title.encode('utf-8')), \ "feed_link": feed_link, \ @@ -149,7 +155,12 @@ class FeedGetter(object): post_date = datetime(*article.published_parsed[:6]) - article = {"article_id": article.link.encode('utf-8'), \ + sha1_hash = hashlib.sha1() + sha1_hash.update(article.link.encode('utf-8')) + article_id = sha1_hash.hexdigest() + + article = {"article_id": article_id, \ + "type":1, \ "article_date": post_date, \ "article_link": article.link.encode('utf-8'), \ "article_title": article_title, \ @@ -160,7 +171,7 @@ class FeedGetter(object): articles.append(article) - self.articles.add_articles(articles, feed_link) + self.articles.add_articles(articles, feed_id) # send new articles by e-mail if desired. #threading.Thread(None, utils.send_mail, None, (utils.mail_from, utils.mail_to, \ diff --git a/mongodb.py b/mongodb.py index 42aef4bb..3faaa318 100644 --- a/mongodb.py +++ b/mongodb.py @@ -9,9 +9,10 @@ __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" import time - import pymongo +from operator import itemgetter, attrgetter + class Articles(object): """ """ @@ -26,25 +27,26 @@ class Articles(object): """ Creates a new collection for a new feed. """ - new_collection["type"] = 0 - - name = str(new_collection["collection_id"]) - pymongo.collection.Collection(self.db, name) - collection = self.db[name] + #pymongo.collection.Collection(self.db, new_collection["feed_id"]) + collection = self.db[new_collection["feed_id"]] collection.create_index([("article_link", pymongo.ASCENDING)], {"unique":True, "sparse":True}) collection.insert(new_collection) - def add_articles(self, articles, collection_id): + def add_articles(self, articles, feed_id): """ Add article(s) in a collection. """ - collection = self.db[str(collection_id)] + collection = self.db[str(feed_id)] for article in articles: - article["type"] = 1 cursor = collection.find({"article_id":article["article_id"]}) if cursor.count() == 0: collection.insert(article) + def get_collection(self, feed_id): + """ + """ + return self.db[str(feed_id)].find().next() + def get_all_articles(self): """ Return all articles from all collections. @@ -56,12 +58,26 @@ class Articles(object): articles.append(collection) return articles - def get_articles_from_collection(self, collection_id): + def get_all_collections(self): + """ + """ + feeds = [] + collections = self.db.collection_names() + for collection_name in collections: + if collection_name != "system.indexes": + cursor = self.db[collection_name].find({"type":0}) + if cursor.count() != 0: + feeds.append(cursor.next()) + feeds = sorted(feeds, key=itemgetter('feed_title')) + return feeds + + def get_articles_from_collection(self, feed_id): """ Return all the articles of a collection. """ - collection = self.db[str(collection_id)] - return collection + collection = self.db[str(feed_id)] + cursor = collection.find({"type":1}) + return cursor.sort([("article_date", pymongo.DESCENDING)]) def print_articles_from_collection(self, collection_id): """ @@ -74,13 +90,34 @@ class Articles(object): print d print - def nb_users(self): + def nb_articles(self, feed_id=None): """ Return the number of users. """ - collection = self.db.users - collection.count() - + if feed_id is not None: + collection = self.db[feed_id] + return collection.find({"type":1}).count() + + def nb_favorites(self): + return "12" + + def nb_mail_notifications(self): + return "42" + + def nb_unread_articles(self, feed_id=None): + if feed_id is not None: + collection = self.db[feed_id] + cursor = collection.find({'article_readed':False}) + return cursor.count() + else: + unread_articles = 0 + for feed_id in self.db.collection_names(): + unread_articles += self.nb_unread_articles(feed_id) + return unread_articles + + + + def list_collections(self): """ List all collections (feed). @@ -137,11 +174,23 @@ if __name__ == "__main__": # Print articles of the collection - articles.print_articles_from_collection("http://esr.ibiblio.org/?feed=rss2") + #articles.print_articles_from_collection("http://esr.ibiblio.org/?feed=rss2") print "All articles:" #print articles.get_all_articles() + + + + + for feed in articles.get_all_collections(): + for article in articles.get_articles_from_collection(feed["feed_id"]): + try: + #print article["article_title"], article["article_date"] + pass + except: + pass + # Drop the database #articles.drop_database() \ No newline at end of file diff --git a/pyAggr3g470r.py b/pyAggr3g470r.py index 28e3ca46..9a88d05a 100755 --- a/pyAggr3g470r.py +++ b/pyAggr3g470r.py @@ -2,7 +2,7 @@ #-*- coding: utf-8 -*- # pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010 Cédric Bonhomme - http://cedricbonhomme.org/ +# Copyright (C) 2010-2012 Cédric Bonhomme - http://cedricbonhomme.org/ # # For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ # @@ -22,7 +22,7 @@ __author__ = "Cedric Bonhomme" __version__ = "$Revision: 3.1 $" __date__ = "$Date: 2010/01/29 $" -__revision__ = "$Date: 2011/11/29 $" +__revision__ = "$Date: 2012/03/03 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" @@ -51,6 +51,7 @@ import datetime import utils import export +import mongodb import feedgetter from qrcode.pyqrnative.PyQRNative import QRCode, QRErrorCorrectLevel, CodeOverflowException from qrcode import qr @@ -107,89 +108,104 @@ class Root: Root class. All pages of pyAggr3g470r are described in this class. """ + def __init__(self): + """ + """ + self.articles = mongodb.Articles() + def index(self): """ Main page containing the list of feeds and articles. """ + feeds = self.articles.get_all_collections() + # if there are unread articles, display the number in the tab of the browser - html = htmlheader((self.nb_unread_articles and \ - ['(' + str(self.nb_unread_articles) +') '] or \ + html = htmlheader((self.articles.nb_unread_articles() and \ + ['(' + str(self.articles.nb_unread_articles()) +') '] or \ [""])[0]) html += htmlnav html += self.create_right_menu() html += """
\n""" - if self.feeds: + if feeds: html += '\n' html += '\n' html += '      \n' html += """\n""" % \ - (self.nb_favorites,) + (self.articles.nb_favorites(),) html += """\n""" % \ - (self.nb_mail_notifications,) + (self.articles.nb_mail_notifications(),) html += '      ' - if self.nb_unread_articles != 0: + if self.articles.nb_unread_articles != 0: html += '\n' html += """\n""" % \ - (self.nb_unread_articles,) + (self.articles.nb_unread_articles(),) html += '\n' + + #for feed in feeds: + #for article in self.articles.get_articles_from_collection(feed["collection_id"]): + #try: + #print article["article_title"], article["article_date"], article["article_readed"] + #except: + #pass + # The main page display all the feeds. - for feed in self.feeds.values(): + for feed in feeds: html += """

%s

\n""" % \ - (feed.feed_id, feed.feed_site_link, feed.feed_title, \ - feed.feed_link, feed.feed_image) + (feed["feed_id"], feed["feed_link"], feed["feed_title"], \ + feed["feed_link"], feed["feed_image"]) # The main page display only 10 articles by feeds. - for article in feed.articles.values()[:10]: - - if article.article_readed == "0": + for article in self.articles.get_articles_from_collection(feed["feed_id"])[:10]: + if article["article_readed"] == False: # not readed articles are in bold not_read_begin, not_read_end = "", "" else: not_read_begin, not_read_end = "", "" # display a heart for faved articles - if article.like == "1": + if article["article_like"] == True: like = """ """ else: like = "" # Descrition for the CSS ToolTips - article_content = utils.clear_string(article.article_description) + article_content = utils.clear_string(article["article_content"]) if article_content: description = " ".join(article_content.split(' ')[:55]) else: description = "No description." # Title of the article - article_title = article.article_title + article_title = article["article_title"] if len(article_title) >= 110: article_title = article_title[:110] + " ..." # a description line per article (date, title of the article and # CSS description tooltips on mouse over) - html += article.article_date + " - " + \ + html += str(article["article_date"]) + " - " + \ """%s%s%s%s""" % \ - (feed.feed_id, article.article_id, not_read_begin, \ + (feed["feed_id"], article["article_id"], not_read_begin, \ article_title, not_read_end, description) + like + "
\n" html += "
\n" # some options for the current feed - html += """All articles   """ % (feed.feed_id,) - html += """Feed summary   """ % (feed.feed_id,) - if feed.nb_unread_articles != 0: - html += """  Mark all as read""" % (feed.feed_id,) - html += """     Unread article(s) (%s)""" % (feed.feed_id, feed.nb_unread_articles) - if feed.mail == "0": - html += """
\nStay tuned""" % (feed.feed_id,) + html += """All articles   """ % (feed["feed_id"],) + html += """Feed summary   """ % (feed["feed_id"],) + feed["nb_unread_articles"] = 0 #hack + if feed["nb_unread_articles"] != 0: + html += """  Mark all as read""" % (feed["feed_id"],) + html += """     Unread article(s) (%s)""" % (feed["feed_id"], feed["nb_unread_articles"]) + if feed["mail"] == "0": + html += """
\nStay tuned""" % (feed["feed_id"],) else: - html += """
\nStop staying tuned""" % (feed.feed_id,) + html += """
\nStop staying tuned""" % (feed["feed_id"],) html += """

Top

""" html += "
\n" html += htmlfooter @@ -215,16 +231,19 @@ class Root: """ Create the list of feeds. """ - html = """
Your feeds (%s):
\n""" % len(self.feeds) - for feed in self.feeds.values(): - if feed.nb_unread_articles != 0: + feeds = self.articles.get_all_collections() + html = """
Your feeds (%s):
\n""" % len(feeds) + for feed in feeds: + feed["nb_unread_articles"] = 0 #hack + if feed["nb_unread_articles"] != 0: # not readed articles are in bold not_read_begin, not_read_end = "", "" else: not_read_begin, not_read_end = "", "" + feed["nb_articles"] = "5" html += """
%s (%s%s%s / %s)
""" % \ - (feed.feed_id, feed.feed_title, feed.feed_id, not_read_begin, \ - feed.nb_unread_articles, not_read_end, feed.nb_articles) + (feed["feed_id"], feed["feed_title"], feed["feed_id"], not_read_begin, \ + self.articles.nb_unread_articles(feed["feed_id"]), not_read_end, self.articles.nb_articles(feed["feed_id"])) return html + "
" @@ -319,8 +338,9 @@ class Root: html += htmlfooter return html - statistics.exposed = True + + def q(self, querystring=None): """ Simply search for the string 'querystring' @@ -550,76 +570,77 @@ class Root: favourite articles for the current feed. """ try: - feed = self.feeds[feed_id] + feed = self.articles.get_collection(feed_id) + articles = self.articles.get_articles_from_collection(feed_id) except KeyError: return self.error_page("This feed do not exists.") html = htmlheader() html += htmlnav html += """
""" - html += "

The feed " + feed.feed_title + " contains " + str(feed.nb_articles) + " articles. " - html += "Representing " + str((round(float(feed.nb_articles) / self.nb_articles, 4)) * 100) + " % of the total " - html += "(" + str(self.nb_articles) + ").

" - if feed.articles.values() != []: - html += "

" + (feed.nb_unread_articles == 0 and ["All articles are read"] or [str(feed.nb_unread_articles) + \ - " unread article" + (feed.nb_unread_articles == 1 and [""] or ["s"])[0]])[0] + ".

" - if feed.mail == "1": + html += "

The feed " + feed["feed_title"] + " contains " + str(self.articles.nb_articles(feed_id)) + " articles. " + html += "Representing " + str((round(float(self.articles.nb_articles(feed_id)) / 1000, 4)) * 100) + " % of the total " #hack + html += "(" + str(1000) + ").

" + if articles != []: + html += "

" + (self.articles.nb_unread_articles(feed_id) == 0 and ["All articles are read"] or [str(self.articles.nb_unread_articles(feed_id)) + \ + " unread article" + (self.articles.nb_unread_articles(feed_id) == 1 and [""] or ["s"])[0]])[0] + ".

" + if feed["mail"] == True: html += """

You are receiving articles from this feed to the address: %s. """ % \ (utils.mail_to, utils.mail_to) html += """Stop receiving articles from this feed.

""" % \ - (feed.feed_id, ) + (feed[feed_id], ) - if feed.articles.values() != []: - last_article = utils.string_to_datetime(feed.articles.values()[0].article_date) - first_article = utils.string_to_datetime(feed.articles.values()[-1].article_date) + if articles != []: + last_article = utils.string_to_datetime(str(articles[0]["article_date"])) + first_article = utils.string_to_datetime(str(articles[self.articles.nb_articles(feed_id)-2]["article_date"])) delta = last_article - first_article delta_today = datetime.datetime.fromordinal(datetime.date.today().toordinal()) - last_article html += "

The last article was posted " + str(abs(delta_today.days)) + " day(s) ago.

" if delta.days > 0: - html += """

Daily average: %s,""" % (str(round(float(feed.nb_articles)/abs(delta.days), 2)),) + html += """

Daily average: %s,""" % (str(round(float(self.articles.nb_articles(feed_id))/abs(delta.days), 2)),) html += """ between the %s and the %s.

\n""" % \ - (feed.articles.values()[-1].article_date[:10], feed.articles.values()[0].article_date[:10]) + (str(articles[self.articles.nb_articles(feed_id)-2]["article_date"])[:10], str(articles[0]["article_date"])[:10]) html += "

Recent articles

" - for article in feed.articles.values()[:10]: - if article.article_readed == "0": + for article in articles[:10]: + if article["article_readed"] == False: # not readed articles are in bold not_read_begin, not_read_end = "", "" else: not_read_begin, not_read_end = "", "" # display a heart for faved articles - if article.like == "1": + if article["article_like"] == True: like = """ """ else: like = "" # Descrition for the CSS ToolTips - article_content = utils.clear_string(article.article_description) + article_content = utils.clear_string(article["article_content"]) if article_content: description = " ".join(article_content[:500].split(' ')[:-1]) else: description = "No description." # Title of the article - article_title = article.article_title + article_title = article["article_title"] if len(article_title) >= 110: article_title = article_title[:110] + " ..." # a description line per article (date, title of the article and # CSS description tooltips on mouse over) - html += article.article_date + " - " + \ + html += str(article["article_date"]) + " - " + \ """%s%s%s%s""" % \ - (feed.feed_id, article.article_id, not_read_begin, \ + (feed["feed_id"], article["article_id"], not_read_begin, \ article_title, not_read_end, description) + like + "
\n" html += "
\n" - html += """All articles   """ % (feed.feed_id,) + html += """All articles   """ % (feed["feed_id"],) - favs = [article for article in feed.articles.values() if article.like == "1"] + favs = [article for article in articles if article["article_like"] == True] if len(favs) != 0: html += "

Your favorites articles for this feed

" for article in favs: - if article.like == "1": + if article["like"] == True: # descrition for the CSS ToolTips - article_content = utils.clear_string(article.article_description) + article_content = utils.clear_string(article["article_content"]) if article_content: description = " ".join(article_content[:500].split(' ')[:-1]) else: @@ -627,9 +648,9 @@ class Root: # a description line per article (date, title of the article and # CSS description tooltips on mouse over) - html += article.article_date + " - " + \ + html += str(article["article_date"]) + " - " + \ """%s%s
\n""" % \ - (feed.feed_id, article.article_id, article.article_title[:150], description) + (feed["feed_id"], article["article_id"], article["article_title"][:150], description) # This section enables the user to edit informations about @@ -642,25 +663,25 @@ class Root: '' + \ """\n""" % \ - (feed.feed_link,) + (feed["feed_link"],) html += '\n\n
' + \ '' + \ """
\n""" % \ - (feed.feed_link,) + (feed["feed_link"],) html += '\n\n
' + \ '' + \ """
\n""" % \ - (feed.feed_link,) + (feed["feed_link"],) dic = {} - dic[feed.feed_id] = self.feeds[feed.feed_id] - top_words = utils.top_words(dic, n=50, size=int(word_size)) + #dic[feed.feed_id] = self.feeds[feed.feed_id] + top_words = utils.top_words([articles], n=50, size=int(word_size)) html += "

Tag cloud

\n
\n" # Tags cloud html += 'Minimum size of a word:' - html += """
""" % (feed.feed_id,) + html += """""" % (feed["feed_id"],) html += """""" % (word_size,) html += '
\n' html += '
' + \ @@ -678,8 +699,9 @@ class Root: """ This page displays all articles of a feed. """ + print "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" try: - feed = self.feeds[feed_id] + feed = self.articles.get_articles_from_collection(feed_id) except KeyError: return self.error_page("This feed do not exists.") html = htmlheader() @@ -690,23 +712,23 @@ class Root: html += "
\n" html += self.create_list_of_feeds() html += """
""" - html += """

Articles of the feed %s


""" % (feed.feed_title) + html += """

Articles of the feed %s


""" % (feed_id,) - for article in feed.articles.values(): + for article in feed: - if article.article_readed == "0": + if article["article_readed"] == False: # not readed articles are in bold not_read_begin, not_read_end = "", "" else: not_read_begin, not_read_end = "", "" - if article.like == "1": + if article["article_like"] == True: like = """ """ else: like = "" # descrition for the CSS ToolTips - article_content = utils.clear_string(article.article_description) + article_content = utils.clear_string(article["content"]) if article_content: description = " ".join(article_content[:500].split(' ')[:-1]) else: @@ -714,10 +736,10 @@ class Root: # a description line per article (date, title of the article and # CSS description tooltips on mouse over) - html += article.article_date + " - " + \ + html += str(article["article_date"]) + " - " + \ """%s%s%s%s""" % \ - (feed.feed_id, article.article_id, not_read_begin, \ - article.article_title[:150], not_read_end, description) + like + "
\n" + (feed_id, article["article_id"], not_read_begin, \ + article["article_title"][:150], not_read_end, description) + like + "
\n" html += """\n

All feeds

""" html += "
\n" @@ -1321,25 +1343,26 @@ if __name__ == '__main__': root = Root() root.favicon_ico = cherrypy.tools.staticfile.handler(filename=os.path.join(utils.path + "/img/favicon.png")) cherrypy.config.update({ 'server.socket_port': 12556, 'server.socket_host': "0.0.0.0"}) - cherrypy.config.update({'error_page.404': error_page_404}) + #cherrypy.config.update({'error_page.404': error_page_404}) _cp_config = {'request.error_response': handle_error} - if not os.path.isfile(utils.sqlite_base): - # create the SQLite base if not exists - print "Creating data base..." - utils.create_base() - # load the informations from base in memory - print "Loading informations from data base..." - root.update() - # launch the available base monitoring method (gamin or classic) - try: - import gamin - thread_watch_base = threading.Thread(None, root.watch_base, None, ()) - except: - print "The gamin module is not installed." - print "The base of feeds will be monitored with the simple method." - thread_watch_base = threading.Thread(None, root.watch_base_classic, None, ()) - thread_watch_base.setDaemon(True) - thread_watch_base.start() + + #if not os.path.isfile(utils.sqlite_base): + ## create the SQLite base if not exists + #print "Creating data base..." + #utils.create_base() + ## load the informations from base in memory + #print "Loading informations from data base..." + #root.update() + ## launch the available base monitoring method (gamin or classic) + #try: + #import gamin + #thread_watch_base = threading.Thread(None, root.watch_base, None, ()) + #except: + #print "The gamin module is not installed." + #print "The base of feeds will be monitored with the simple method." + #thread_watch_base = threading.Thread(None, root.watch_base_classic, None, ()) + #thread_watch_base.setDaemon(True) + #thread_watch_base.start() cherrypy.quickstart(root, "/" ,config=utils.path + "/cfg/cherrypy.cfg") diff --git a/utils.py b/utils.py index 0f5453c3..e44239e4 100755 --- a/utils.py +++ b/utils.py @@ -171,9 +171,9 @@ def top_words(feeds, n=10, size=5): """ words = Counter() wordre = re.compile(r'\b\w{%s,}\b' % size, re.I) - for feed in feeds.values(): - for article in feed.articles.values(): - for word in wordre.findall(clear_string(article.article_description)): + for feed in feeds: + for article in feed: + for word in wordre.findall(clear_string(article["article_content"])): words[word.lower()] += 1 return words.most_common(n) -- cgit