diff options
Diffstat (limited to 'source')
74 files changed, 0 insertions, 4882 deletions
diff --git a/source/auth.py b/source/auth.py deleted file mode 100755 index 82c3a440..00000000 --- a/source/auth.py +++ /dev/null @@ -1,269 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.3 $" -__date__ = "$Date: 2012/10/12 $" -__revision__ = "$Date: 2013/01/14 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -# -# Form based authentication for CherryPy. Requires the -# Session tool to be loaded. -# - -import cherrypy -import hashlib - -import log - -SESSION_KEY = '_cp_username' - -import csv -class excel_french(csv.Dialect): - delimiter = ';' - quotechar = '"' - doublequote = True - skipinitialspace = False - lineterminator = '\n' - quoting = csv.QUOTE_MINIMAL - -csv.register_dialect('excel_french', excel_french) - -def change_username(username, new_username, password_file='./var/password'): - """ - Change the password corresponding to username. - """ - users_list = [] - result = False - with open(password_file, 'r') as csv_readfile_read: - cr = csv.reader(csv_readfile_read, 'excel_french') - users_list = [elem for elem in cr] - with open(password_file, 'w') as csv_file_write: - cw = csv.writer(csv_file_write, 'excel_french') - for user in users_list: - if user[0] == username: - cw.writerow([new_username, user[1]]) - result = True - else: - cw.writerow(user) - return result - -def change_password(username, new_password, password_file='./var/password'): - """ - Change the password corresponding to username. - """ - users_list = [] - result = False - with open(password_file, 'r') as csv_readfile_read: - cr = csv.reader(csv_readfile_read, 'excel_french') - users_list = [elem for elem in cr] - with open(password_file, 'w') as csv_file_write: - cw = csv.writer(csv_file_write, 'excel_french') - for user in users_list: - if user[0] == username: - m = hashlib.sha1() - m.update(new_password.encode()) - cw.writerow([user[0], m.hexdigest()]) - result = True - else: - cw.writerow(user) - return result - -def check_credentials(username, password, password_file='./var/password'): - """ - Verifies credentials for username and password. - Returns None on success or a string describing the error on failure. - """ - USERS = {} - cr = csv.reader(open(password_file, "r"), 'excel_french') - for row in cr: - USERS[row[0]] = row[1] - - m = hashlib.sha1() - m.update(password.encode()) - if username in list(USERS.keys()) and USERS[username] == m.hexdigest(): - return None - else: - return "Incorrect username or password." - # An example implementation which uses an ORM could be: - # u = User.get(username) - # if u is None: - # return u"Username %s is unknown to me." % username - # if u.password != md5.new(password).hexdigest(): - # return u"Incorrect password" - -def check_auth(*args, **kwargs): - """ - A tool that looks in config for 'auth.require'. If found and it - is not None, a login is required and the entry is evaluated as a list of - conditions that the user must fulfill. - """ - conditions = cherrypy.request.config.get('auth.require', None) - if conditions is not None: - username = cherrypy.session.get(SESSION_KEY) - if username: - cherrypy.request.login = username - for condition in conditions: - # A condition is just a callable that returns true or false - if not condition(): - raise cherrypy.HTTPRedirect("/auth/login") - else: - raise cherrypy.HTTPRedirect("/auth/login") - -cherrypy.tools.auth = cherrypy.Tool('before_handler', check_auth) - -def require(*conditions): - """ - A decorator that appends conditions to the auth.require config - variable. - """ - def decorate(f): - if not hasattr(f, '_cp_config'): - f._cp_config = dict() - if 'auth.require' not in f._cp_config: - f._cp_config['auth.require'] = [] - f._cp_config['auth.require'].extend(conditions) - return f - return decorate - - -# Conditions are callables that return True -# if the user fulfills the conditions they define, False otherwise -# -# They can access the current username as cherrypy.request.login -# -# Define those at will however suits the application. - -def member_of(groupname): - def check(): - # replace with actual check if <username> is in <groupname> - return cherrypy.request.login == 'joe' and groupname == 'admin' - return check - -def name_is(reqd_username): - return lambda: reqd_username == cherrypy.request.login - -# These might be handy - -def any_of(*conditions): - """ - Returns True if any of the conditions match. - """ - def check(): - for c in conditions: - if c(): - return True - return False - return check - -# By default all conditions are required, but this might still be -# needed if you want to use it inside of an any_of(...) condition -def all_of(*conditions): - """ - Returns True if all of the conditions match. - """ - def check(): - for c in conditions: - if not c(): - return False - return True - return check - - -class AuthController(object): - """ - This class provides login and logout actions. - """ - def __init__(self): - self.logger = log.Log() - self.username = "" - - def on_login(self, username): - """ - Called on successful login. - """ - self.username = username - self.logger.info(username + ' logged in.') - - def on_logout(self, username): - """ - Called on logout. - """ - self.logger.info(username + ' logged out.') - self.username = "" - - def get_loginform(self, username, msg="Enter login information", from_page="/"): - """ - Login page. - """ - return """<!DOCTYPE html>\n<html> - <head> - <meta charset="utf-8" /> - <title>pyAggr3g470r</title> - <link rel="stylesheet" href="/css/log.css" /> - </head> - <body> - <div> - <div id="logform"> - <img src="/static/img/tuxrss.png" alt="pyAggr3g470r" /> - <form method="post" action="/auth/login"> - <input type="hidden" name="from_page" value="%(from_page)s" /> - %(msg)s<br /> - <input type="text" name="username" value="%(username)s" placeholder="Username" autofocus="autofocus" /><br /> - <input type="password" name="password" placeholder="Password" /><br /> - <input type="submit" value="Log in" /> - </form> - </div><!-- end #main --> - </div><!-- end #center --> - </body> -</html>""" % locals() - - @cherrypy.expose - def login(self, username=None, password=None, from_page="/"): - """ - Open a session for an authenticated user. - """ - if username is None or password is None: - return self.get_loginform("", from_page=from_page) - - error_msg = check_credentials(username, password) - if error_msg: - self.logger.info(error_msg) - return self.get_loginform(username, error_msg, from_page) - else: - cherrypy.session[SESSION_KEY] = cherrypy.request.login = username - self.on_login(username) - raise cherrypy.HTTPRedirect(from_page or "/") - - @cherrypy.expose - def logout(self, from_page="/"): - """ - Cloase a session. - """ - sess = cherrypy.session - username = sess.get(SESSION_KEY, None) - sess[SESSION_KEY] = None - if username: - cherrypy.request.login = None - self.on_logout(username) - raise cherrypy.HTTPRedirect(from_page or "/")
\ No newline at end of file diff --git a/source/binarytree.py b/source/binarytree.py deleted file mode 100644 index a9294251..00000000 --- a/source/binarytree.py +++ /dev/null @@ -1,177 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -""" -A binary ordered tree implementation. -""" - -class Node(object): - """ - Represents a node. - """ - def __init__(self, data): - """ - Initialization. - """ - self.left = None - self.right = None - self.data = data - -class OrderedBinaryTree(object): - """ - Represents a binary ordered . - """ - def __init__(self, root=None): - """ - Initializes the root member. - """ - self.root = root - - def addNode(self, data): - """ - Creates a new node and returns it. - """ - return Node(data) - - def insert(self, root, data): - """ - Inserts a new data. - """ - if root == None: - # it there isn't any data - # adds it and returns - return self.addNode(data) - else: - # enters into the - if data['article_date'] <= root.data['article_date']: - # if the data is less than the stored one - # goes into the left-sub- - root.left = self.insert(root.left, data) - else: - # processes the right-sub- - root.right = self.insert(root.right, data) - return root - - def lookup(self, root, target): - """ - Looks for a value into the . - """ - if root == None: - return 0 - else: - # if it has found it... - if target == root.data: - return 1 - else: - if target['article_date'] < root.data['article_date']: - # left side - return self.lookup(root.left, target) - else: - # right side - return self.lookup(root.right, target) - - def minValue(self, root): - """ - Goes down into the left - arm and returns the last value. - """ - while(root.left != None): - root = root.left - return root.data - - def maxValue(self, root): - """ - Goes down into the right - arm and returns the last value. - """ - while(root.right != None): - root = root.right - return root.data - - def maxDepth(self, root): - """ - Return the maximum depth. - """ - if root == None: - return 0 - else: - # computes the two depths - ldepth = self.maxDepth(root.left) - rdepth = self.maxDepth(root.right) - # returns the appropriate depth - return max(ldepth, rdepth) + 1 - - def size(self, root): - if root == None: - return 0 - else: - return self.size(root.left) + 1 + self.size(root.right) - - def pre_order_traversal(self, root, result=[]): - """ - Depth-first. Pre-order traversal. - """ - if root == None: - pass - else: - result.append(root.data) - self.pre_order_traversal(root.left, result) - self.pre_order_traversal(root.right, result) - return result - - def in_order_traversal(self, root, result=[]): - """ - Depth-first. In-order traversal. - """ - if root == None: - pass - else: - self.in_order_traversal(root.left, result) - result.append(root.data) - self.in_order_traversal(root.right, result) - return result - - def post_order_traversal(self, root, result=[]): - """ - Depth-first. Post-order traversal. - """ - if root == None: - pass - else: - self.post_order_traversal(root.left, result) - self.post_order_traversal(root.right, result) - result.append(root.data) - return result - - def __str__(self): - """ - Pretty display. - """ - return ", ".join([article["article_title"] for article in \ - self.in_order_traversal(self.root)]) - -if __name__ == "__main__": - # Point of entry in execution mode. - # create the tree - tree = OrderedBinaryTree() - # add the root node - root = tree.addNode(0) - # ask the user to insert values - for i in range(0, 5): - data = int(input("insert the node value nr %d: " % i)) - # insert values - tree.insert(root, data) - - tree.printTree(root) - print() - tree.printRevTree(root) - print() - data = int(input("Insert a value to find: ")) - if tree.lookup(root, data): - print("found") - else: - print("not found") - - print(tree.minValue(root)) - print(tree.maxDepth(root)) - print(tree.size(root))
\ No newline at end of file diff --git a/source/cfg/cherrypy.cfg b/source/cfg/cherrypy.cfg deleted file mode 100644 index ac05d330..00000000 --- a/source/cfg/cherrypy.cfg +++ /dev/null @@ -1,26 +0,0 @@ -[global] -server.socket_host: "0.0.0.0" -server.socket_port: 12556 -server.environment = "production" -engine.autoreload_on = True -engine.autoreload_frequency = 5 -engine.timeout_monitor.on = False -log.error_file = "var/error.log" -log.access_file = "var/access.log" - -[/] -tools.staticdir.root = os.getcwd() -tools.staticdir.on = True -tools.staticdir.dir = "." -tools.encode.on = True -tools.encode.encoding = "utf8" - -[/css] -tools.staticdir.on = True -tools.staticdir.dir = "static/css" -tools.staticdir.match = "(?i)^.+\.css$" - -[/images] -tools.staticdir.on = True -tools.staticdir.dir = "static/img" -tools.staticdir.match = "(?i)^.+\.png$"
\ No newline at end of file diff --git a/source/cfg/pyAggr3g470r.cfg-sample b/source/cfg/pyAggr3g470r.cfg-sample deleted file mode 100755 index f22e2850..00000000 --- a/source/cfg/pyAggr3g470r.cfg-sample +++ /dev/null @@ -1,19 +0,0 @@ -[MongoDB] -address = 127.0.0.1 -port = 27017 -dbname = bob_pyaggr3g470r -user = bob -password = -[feedparser] -http_proxy = 127.0.0.1:8118 -user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r) -feed_list = ./var/feed.lst -[mail] -enabled = 0 -mail_from = pyAggr3g470r@no-reply.com -mail_to = address_of_the_recipient@example.com -smtp = smtp.example.com -username = your_mail_address@example.com -password = your_password -[misc] -diaspora_pod = joindiaspora.com
\ No newline at end of file diff --git a/source/conf.py b/source/conf.py deleted file mode 100644 index 1b262927..00000000 --- a/source/conf.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2012/04/22 $" -__revision__ = "$Date: 2013/08/15 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - - -import os -import configparser -# load the configuration -config = configparser.SafeConfigParser() -try: - config.read("./cfg/pyAggr3g470r.cfg") -except: - config.read("./cfg/pyAggr3g470r.cfg-sample") -path = os.path.abspath(".") - -MONGODB_ADDRESS = config.get('MongoDB', 'address') -MONGODB_PORT = int(config.get('MongoDB', 'port')) -MONGODB_DBNAME = config.get('MongoDB', 'dbname') -MONGODB_USER = config.get('MongoDB', 'user') -MONGODB_PASSWORD = config.get('MongoDB', 'password') - -HTTP_PROXY = config.get('feedparser', 'http_proxy') -USER_AGENT = config.get('feedparser', 'user_agent') -FEED_LIST = config.get('feedparser', 'feed_list') - -MAIL_ENABLED = bool(int(config.get('mail','enabled'))) -mail_from = config.get('mail','mail_from') -mail_to = config.get('mail','mail_to') -smtp_server = config.get('mail','smtp') -username = config.get('mail','username') -password = config.get('mail','password') - -DIASPORA_POD = config.get('misc', 'diaspora_pod')
\ No newline at end of file diff --git a/source/epub/__init__.py b/source/epub/__init__.py deleted file mode 100644 index 8d1c8b69..00000000 --- a/source/epub/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/source/epub/epub.py b/source/epub/epub.py deleted file mode 100644 index 2c01b54a..00000000 --- a/source/epub/epub.py +++ /dev/null @@ -1,343 +0,0 @@ -#! /usr/local/bin/python
-#-*- coding: utf-8 -*-
-
-import itertools
-import mimetypes
-import os
-import shutil
-import subprocess
-import uuid
-import zipfile
-from genshi.template import TemplateLoader
-from lxml import etree
-
-class TocMapNode:
- def __init__(self):
- self.playOrder = 0
- self.title = ''
- self.href = ''
- self.children = []
- self.depth = 0
-
- def assignPlayOrder(self):
- nextPlayOrder = [0]
- self.__assignPlayOrder(nextPlayOrder)
-
- def __assignPlayOrder(self, nextPlayOrder):
- self.playOrder = nextPlayOrder[0]
- nextPlayOrder[0] = self.playOrder + 1
- for child in self.children:
- child.__assignPlayOrder(nextPlayOrder)
-
-class EpubItem:
- def __init__(self):
- self.id = ''
- self.srcPath = ''
- self.destPath = ''
- self.mimeType = ''
- self.html = ''
-
-class EpubBook:
- def __init__(self):
- self.loader = TemplateLoader('./epub/templates')
-
- self.rootDir = ''
- self.UUID = uuid.uuid1()
-
- self.lang = 'en-US'
- self.title = ''
- self.creators = []
- self.metaInfo = []
-
- self.imageItems = {}
- self.htmlItems = {}
- self.cssItems = {}
-
- self.coverImage = None
- self.titlePage = None
- self.tocPage = None
-
- self.spine = []
- self.guide = {}
- self.tocMapRoot = TocMapNode()
- self.lastNodeAtDepth = {0 : self.tocMapRoot}
-
- def setTitle(self, title):
- self.title = title
-
- def setLang(self, lang):
- self.lang = lang
-
- def addCreator(self, name, role = 'aut'):
- self.creators.append((name, role))
-
- def addMeta(self, metaName, metaValue, **metaAttrs):
- self.metaInfo.append((metaName, metaValue, metaAttrs))
-
- def getMetaTags(self):
- l = []
- for metaName, metaValue, metaAttr in self.metaInfo:
- beginTag = '<dc:%s' % metaName
- if metaAttr:
- for attrName, attrValue in metaAttr.iteritems():
- beginTag += ' opf:%s="%s"' % (attrName, attrValue)
- beginTag += '>'
- endTag = '</dc:%s>' % metaName
- l.append((beginTag, metaValue, endTag))
- return l
-
- def getImageItems(self):
- return sorted(self.imageItems.values(), key = lambda x : x.id)
-
- def getHtmlItems(self):
- return sorted(self.htmlItems.values(), key = lambda x : x.id)
-
- def getCssItems(self):
- return sorted(self.cssItems.values(), key = lambda x : x.id)
-
- def getAllItems(self):
- return sorted(itertools.chain(self.imageItems.values(), self.htmlItems.values(), self.cssItems.values()), key = lambda x : x.id)
-
- def addImage(self, srcPath, destPath):
- item = EpubItem()
- item.id = 'image_%d' % (len(self.imageItems) + 1)
- item.srcPath = srcPath
- item.destPath = destPath
- item.mimeType = mimetypes.guess_type(destPath)[0]
- assert item.destPath not in self.imageItems
- self.imageItems[destPath] = item
- return item
-
- def addHtmlForImage(self, imageItem):
- tmpl = self.loader.load('image.html')
- stream = tmpl.generate(book = self, item = imageItem)
- html = stream.render('xhtml', doctype = 'xhtml11', drop_xml_decl = False)
- return self.addHtml('', '%s.html' % imageItem.destPath, html)
-
- def addHtml(self, srcPath, destPath, html):
- item = EpubItem()
- item.id = 'html_%d' % (len(self.htmlItems) + 1)
- item.srcPath = srcPath
- item.destPath = destPath
- item.html = html
- item.mimeType = 'application/xhtml+xml'
- assert item.destPath not in self.htmlItems
- self.htmlItems[item.destPath] = item
- return item
-
- def addCss(self, srcPath, destPath):
- item = EpubItem()
- item.id = 'css_%d' % (len(self.cssItems) + 1)
- item.srcPath = srcPath
- item.destPath = destPath
- item.mimeType = 'text/css'
- assert item.destPath not in self.cssItems
- self.cssItems[item.destPath] = item
- return item
-
- def addCover(self, srcPath):
- assert not self.coverImage
- _, ext = os.path.splitext(srcPath)
- destPath = 'cover%s' % ext
- self.coverImage = self.addImage(srcPath, destPath)
- #coverPage = self.addHtmlForImage(self.coverImage)
- #self.addSpineItem(coverPage, False, -300)
- #self.addGuideItem(coverPage.destPath, 'Cover', 'cover')
-
- def __makeTitlePage(self):
- assert self.titlePage
- if self.titlePage.html:
- return
- tmpl = self.loader.load('title-page.html')
- stream = tmpl.generate(book = self)
- self.titlePage.html = stream.render('xhtml', doctype = 'xhtml11', drop_xml_decl = False)
-
- def addTitlePage(self, html = ''):
- assert not self.titlePage
- self.titlePage = self.addHtml('', 'title-page.html', html)
- self.addSpineItem(self.titlePage, True, -200)
- self.addGuideItem('title-page.html', 'Title Page', 'title-page')
-
- def __makeTocPage(self):
- assert self.tocPage
- tmpl = self.loader.load('toc.html')
- stream = tmpl.generate(book = self)
- self.tocPage.html = stream.render('xhtml', doctype = 'xhtml11', drop_xml_decl = False)
-
- def addTocPage(self):
- assert not self.tocPage
- self.tocPage = self.addHtml('', 'toc.html', '')
- self.addSpineItem(self.tocPage, False, -100)
- self.addGuideItem('toc.html', 'Table of Contents', 'toc')
-
- def getSpine(self):
- return sorted(self.spine)
-
- def addSpineItem(self, item, linear = True, order = None):
- assert item.destPath in self.htmlItems
- if order == None:
- order = (max(order for order, _, _ in self.spine) if self.spine else 0) + 1
- self.spine.append((order, item, linear))
-
- def getGuide(self):
- return sorted(self.guide.values(), key = lambda x : x[2])
-
- def addGuideItem(self, href, title, type):
- assert type not in self.guide
- self.guide[type] = (href, title, type)
-
- def getTocMapRoot(self):
- return self.tocMapRoot
-
- def getTocMapHeight(self):
- return max(self.lastNodeAtDepth.keys())
-
- def addTocMapNode(self, href, title, depth = None, parent = None):
- node = TocMapNode()
- node.href = href
- node.title = title
- if parent == None:
- if depth == None:
- parent = self.tocMapRoot
- else:
- parent = self.lastNodeAtDepth[depth - 1]
- parent.children.append(node)
- node.depth = parent.depth + 1
- self.lastNodeAtDepth[node.depth] = node
- return node
-
- def makeDirs(self):
- try:
- os.makedirs(os.path.join(self.rootDir, 'META-INF'))
- except OSError:
- pass
- try:
- os.makedirs(os.path.join(self.rootDir, 'OEBPS'))
- except OSError:
- pass
-
- def __writeContainerXML(self):
- fout = open(os.path.join(self.rootDir, 'META-INF', 'container.xml'), 'w')
- tmpl = self.loader.load('container.xml')
- stream = tmpl.generate()
- fout.write(stream.render('xml'))
- fout.close()
-
- def __writeTocNCX(self):
- self.tocMapRoot.assignPlayOrder()
- fout = open(os.path.join(self.rootDir, 'OEBPS', 'toc.ncx'), 'w')
- tmpl = self.loader.load('toc.ncx')
- stream = tmpl.generate(book = self)
- fout.write(stream.render('xml'))
- fout.close()
-
- def __writeContentOPF(self):
- fout = open(os.path.join(self.rootDir, 'OEBPS', 'content.opf'), 'w')
- tmpl = self.loader.load('content.opf')
- stream = tmpl.generate(book = self)
- fout.write(stream.render('xml'))
- fout.close()
-
- def __writeItems(self):
- for item in self.getAllItems():
- #print item.id, item.destPath
- if item.html:
- fout = open(os.path.join(self.rootDir, 'OEBPS', item.destPath), 'w')
- fout.write(item.html)
- fout.close()
- else:
- shutil.copyfile(item.srcPath, os.path.join(self.rootDir, 'OEBPS', item.destPath))
-
-
- def __writeMimeType(self):
- fout = open(os.path.join(self.rootDir, 'mimetype'), 'w')
- fout.write('application/epub+zip')
- fout.close()
-
- @staticmethod
- def __listManifestItems(contentOPFPath):
- tree = etree.parse(contentOPFPath)
- return tree.xpath("//opf:manifest/opf:item/@href", namespaces = {'opf': 'http://www.idpf.org/2007/opf'})
-
- @staticmethod
- def createArchive(rootDir, outputPath):
- fout = zipfile.ZipFile(outputPath, 'w')
- cwd = os.getcwd()
- os.chdir(rootDir)
- fout.write('mimetype', compress_type = zipfile.ZIP_STORED)
- fileList = []
- fileList.append(os.path.join('META-INF', 'container.xml'))
- fileList.append(os.path.join('OEBPS', 'content.opf'))
- for itemPath in EpubBook.__listManifestItems(os.path.join('OEBPS', 'content.opf')):
- fileList.append(os.path.join('OEBPS', itemPath))
- for filePath in fileList:
- fout.write(filePath, compress_type = zipfile.ZIP_DEFLATED)
- fout.close()
- os.chdir(cwd)
-
- def createBook(self, rootDir):
- if self.titlePage:
- self.__makeTitlePage()
- if self.tocPage:
- self.__makeTocPage()
- self.rootDir = rootDir
- self.makeDirs()
- self.__writeMimeType()
- self.__writeItems()
- self.__writeContainerXML()
- self.__writeContentOPF()
- self.__writeTocNCX()
-
-def test():
- def getMinimalHtml(text):
- return """<!DOCTYPE html PUBLIC "-//W3C//DTD XHtml 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head><title>%s</title></head>
-<body><p>%s</p></body>
-</html>
-""" % (text, text)
-
- book = EpubBook()
- book.setTitle('Most Wanted Tips for Aspiring Young Pirates')
- book.addCreator('Monkey D Luffy')
- book.addCreator('Guybrush Threepwood')
- book.addMeta('contributor', 'Smalltalk80', role = 'bkp')
- book.addMeta('date', '2010', event = 'publication')
-
- book.addTitlePage()
- book.addTocPage()
- book.addCover(r'D:\epub\blank.png')
-
- book.addCss(r'main.css', 'main.css')
-
- n1 = book.addHtml('', '1.html', getMinimalHtml('Chapter 1'))
- n11 = book.addHtml('', '2.html', getMinimalHtml('Section 1.1'))
- n111 = book.addHtml('', '3.html', getMinimalHtml('Subsection 1.1.1'))
- n12 = book.addHtml('', '4.html', getMinimalHtml('Section 1.2'))
- n2 = book.addHtml('', '5.html', getMinimalHtml('Chapter 2'))
-
- book.addSpineItem(n1)
- book.addSpineItem(n11)
- book.addSpineItem(n111)
- book.addSpineItem(n12)
- book.addSpineItem(n2)
-
- # You can use both forms to add TOC map
- #t1 = book.addTocMapNode(n1.destPath, '1')
- #t11 = book.addTocMapNode(n11.destPath, '1.1', parent = t1)
- #t111 = book.addTocMapNode(n111.destPath, '1.1.1', parent = t11)
- #t12 = book.addTocMapNode(n12.destPath, '1.2', parent = t1)
- #t2 = book.addTocMapNode(n2.destPath, '2')
-
- book.addTocMapNode(n1.destPath, '1')
- book.addTocMapNode(n11.destPath, '1.1', 2)
- book.addTocMapNode(n111.destPath, '1.1.1', 3)
- book.addTocMapNode(n12.destPath, '1.2', 2)
- book.addTocMapNode(n2.destPath, '2')
-
- rootDir = r'd:\epub\test'
- book.createBook(rootDir)
- EpubBook.createArchive(rootDir, rootDir + '.epub')
-
-if __name__ == '__main__':
- test()
\ No newline at end of file diff --git a/source/epub/ez_epub.py b/source/epub/ez_epub.py deleted file mode 100644 index ecfd4f5a..00000000 --- a/source/epub/ez_epub.py +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/local/bin/python
-#-*- coding: utf-8 -*-
-
-import epub
-from genshi.template import TemplateLoader
-
-class Section:
- def __init__(self):
- self.title = ''
- self.paragraphs = []
- self.tocDepth = 1
-
-def makeBook(title, authors, sections, outputDir, lang='en-US', cover=None):
- book = epub.EpubBook()
- book.setLang(lang)
- book.setTitle(title)
- for author in authors:
- book.addCreator(author)
- #book.addTitlePage()
- #book.addTocPage()
- #if cover:
- #book.addCover(cover)
-
- loader = TemplateLoader('./epub/templates')
- tmpl = loader.load('ez-section.html')
-
- for i, section in enumerate(sections):
- stream = tmpl.generate(section = section)
- html = stream.render('xhtml', doctype='xhtml11', drop_xml_decl=False)
- item = book.addHtml('', 's%d.html' % (i + 1), html)
- book.addSpineItem(item)
- book.addTocMapNode(item.destPath, section.title, section.tocDepth)
-
- outputFile = outputDir + 'article.epub'
- book.createBook(outputDir)
- book.createArchive(outputDir, outputFile)
\ No newline at end of file diff --git a/source/epub/templates/container.xml b/source/epub/templates/container.xml deleted file mode 100644 index eecf7a0d..00000000 --- a/source/epub/templates/container.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0"> - <rootfiles> - <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/> - </rootfiles> -</container> diff --git a/source/epub/templates/content.opf b/source/epub/templates/content.opf deleted file mode 100644 index 67f3f5c6..00000000 --- a/source/epub/templates/content.opf +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" encoding="utf-8" standalone="no"?> -<opf:package xmlns:opf="http://www.idpf.org/2007/opf" - xmlns:dc="http://purl.org/dc/elements/1.1/" - xmlns:py="http://genshi.edgewall.org/" - unique-identifier="bookid" version="2.0"> - <opf:metadata > - <dc:identifier id="bookid">urn:uuid:${book.UUID}</dc:identifier> - <dc:language>${book.lang}</dc:language> - <dc:title>${book.title}</dc:title> - <py:for each="name, role in book.creators"> - <dc:creator opf:role="$role">$name</dc:creator> - </py:for> - <py:for each="beginTag, content, endTag in book.getMetaTags()"> - ${Markup(beginTag)}$content${Markup(endTag)} - </py:for> - <opf:meta name="cover" content="${book.coverImage.id}" py:if="book.coverImage"/> - </opf:metadata> - <opf:manifest> - <opf:item id="ncxtoc" media-type="application/x-dtbncx+xml" href="toc.ncx"/> - <py:for each="item in book.getAllItems()"> - <opf:item id="${item.id}" media-type="${item.mimeType}" href="${item.destPath}"/> - </py:for> - </opf:manifest> - <opf:spine toc="ncxtoc"> - <py:for each="_, item, linear in book.getSpine()"> - <opf:itemref idref="${item.id}" linear="${'yes' if linear else 'no'}"/> - </py:for> - </opf:spine> - <opf:guide py:if="book.guide"> - <py:for each="href, title, type in book.getGuide()"> - <opf:reference href="$href" type="$type" title="$title"/> - </py:for> - </opf:guide> -</opf:package> diff --git a/source/epub/templates/ez-section.html b/source/epub/templates/ez-section.html deleted file mode 100644 index 0a715e7f..00000000 --- a/source/epub/templates/ez-section.html +++ /dev/null @@ -1,17 +0,0 @@ -<html xmlns="http://www.w3.org/1999/xhtml" - xmlns:py="http://genshi.edgewall.org/"> -<head> - <title>${section.title}</title> - <style type="text/css"> -h1 { - text-align: center; -} - </style> -</head> -<body> - <h1>${section.title}</h1> - <py:for each="p in section.paragraphs"> - <p>$p</p> - </py:for> -</body> -</html> diff --git a/source/epub/templates/image.html b/source/epub/templates/image.html deleted file mode 100644 index 9a838c7e..00000000 --- a/source/epub/templates/image.html +++ /dev/null @@ -1,16 +0,0 @@ -<html xmlns="http://www.w3.org/1999/xhtml" - xmlns:py="http://genshi.edgewall.org/"> -<head> - <title>${item.destPath}</title> - <style type="text/css"> -div, img { - border: 0; - margin: 0; - padding: 0; -} - </style> -</head> -<body> - <div><img src="${item.destPath}" alt="${item.destPath}"/></div> -</body> -</html> diff --git a/source/epub/templates/title-page.html b/source/epub/templates/title-page.html deleted file mode 100644 index de0f55f0..00000000 --- a/source/epub/templates/title-page.html +++ /dev/null @@ -1,22 +0,0 @@ -<html xmlns="http://www.w3.org/1999/xhtml" - xmlns:py="http://genshi.edgewall.org/"> -<head> - <title>${book.title}</title> - <style type="text/css"> -.title, .authors { - text-align: center; -} -span.author { - margin: 1em; -} - </style> -</head> -<body> - <h1 class="title">${book.title}</h1> - <h3 class="authors"> - <py:for each="creator, _ in book.creators"> - <span class="author">$creator</span> - </py:for> - </h3> -</body> -</html> diff --git a/source/epub/templates/toc.html b/source/epub/templates/toc.html deleted file mode 100644 index b14c9da3..00000000 --- a/source/epub/templates/toc.html +++ /dev/null @@ -1,32 +0,0 @@ -<html xmlns="http://www.w3.org/1999/xhtml" - xmlns:py="http://genshi.edgewall.org/"> -<head> - <title>${book.title}</title> - <style type="text/css"> -.tocEntry-1 { -} -.tocEntry-2 { - text-indent: 1em; -} -.tocEntry-3 { - text-indent: 2em; -} -.tocEntry-4 { - text-indent: 3em; -} - </style> -</head> -<body> - <py:def function="tocEntry(node)"> - <div class="tocEntry-${node.depth}"> - <a href="${node.href}">${node.title}</a> - </div> - <py:for each="child in node.children"> - ${tocEntry(child)} - </py:for> - </py:def> - <py:for each="child in book.getTocMapRoot().children"> - ${tocEntry(child)} - </py:for> -</body> -</html> diff --git a/source/epub/templates/toc.ncx b/source/epub/templates/toc.ncx deleted file mode 100644 index e7dd391a..00000000 --- a/source/epub/templates/toc.ncx +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" - xmlns:py="http://genshi.edgewall.org/" - version="2005-1"> - <head> - <meta name="dtb:uid" content="urn:uuid:${book.UUID}"/> - <meta name="dtb:depth" content="${book.getTocMapHeight()}"/> - <meta name="dtb:totalPageCount" content="0"/> - <meta name="dtb:maxPageNumber" content="0"/> - </head> - <docTitle> - <text>${book.title}</text> - </docTitle> - <navMap> - <py:def function="navPoint(node)"> - <navPoint id="navPoint-${node.playOrder}" playOrder="${node.playOrder}"> - <navLabel><text>${node.title}</text></navLabel> - <content src="${node.href}"/> - <py:for each="child in node.children"> - ${navPoint(child)} - </py:for> - </navPoint> - </py:def> - <py:for each="child in book.getTocMapRoot().children"> - ${navPoint(child)} - </py:for> - </navMap> -</ncx> diff --git a/source/export.py b/source/export.py deleted file mode 100644 index 5e06aea1..00000000 --- a/source/export.py +++ /dev/null @@ -1,274 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2011/10/24 $" -__revision__ = "$Date: 2013/03/05 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -# -# This file contains the export functions of pyAggr3g470r. Indeed -# it is possible to export the database of articles in different formats: -# - simple HTML webzine; -# - text file; -# - ePub file; -# - PDF file. -# - -import os -import time - -import conf -import utils - -def HTML_HEADER(title="pyAggr3g470r", css="./style.css"): - return """<!DOCTYPE html> -<html lang="en-US"> -<head> -<title>%s</title> -<meta charset="utf-8"/> -<link rel="stylesheet" href="%s" /> -</head> -<body>""" % (title, css) - -HTML_FOOTER = """<hr /> -<p>This archive has been generated with -<a href="https://bitbucket.org/cedricbonhomme/pyaggr3g470r/">pyAggr3g470r</a>. -A software under GPLv3 license. -You are welcome to copy, modify or redistribute the source code according to the -<a href="http://www.gnu.org/licenses/gpl-3.0.txt">GPLv3</a> license.</p> -</body> -</html> -""" - -CSS = """body { - font:normal medium 'Gill Sans','Gill Sans MT',Verdana,sans-serif; - margin:1.20em auto; - width:80%; - line-height:1.75; -} -blockquote { - font-size:small; - line-height:2.153846; - margin:2.153846em 0; - padding:0;font-style:oblique; - border-left:1px dotted; - margin-left:2.153846em; - padding-left:2.153846em; -} -blockquote p{ - margin:2.153846em 0; -} -p+br { - display:none; -} -h1 { -font-size:large; -} -h2,h3 { - font-size:medium; -} -hr { - border-style:dotted; - height:1px; - border-width: 1px 0 0 0; - margin:1.45em 0 1.4em; - padding:0; -} -a { - text-decoration:none; - color:#00008B; -} -#footer { - clear:both; - text-align:center; - font-size:small; -} -img { - border:0; -} -.horizontal,.simple li { - margin:0; - padding:0; - list-style:none; - display:inline -} -.simple li:before { - content:"+ "; -} -.simple > li:first-child:before { - content:""; -} -.author { - text-decoration:none; - display:block; - float:right; - margin-left:2em; - font-size:small; -} -.content { - margin:1.00em 1.00em; -}""" - -def export_html(mongo_db): - """ - Export the articles given in parameter in a simple Webzine. - """ - nb_articles = format(mongo_db.nb_articles(), ",d") - feeds = mongo_db.get_all_feeds() - index = HTML_HEADER("News archive") - index += "<h1>List of feeds</h1>\n" - index += """<p>%s articles.</p>\n<ul>\n""" % (nb_articles,) - for feed in feeds: - # creates a folder for each stream - feed_folder = conf.path + "/var/export/webzine/" + \ - utils.normalize_filename(feed["feed_id"]) - try: - os.makedirs(feed_folder) - except OSError: - # directories already exists (not a problem) - pass - - index += """ <li><a href="%s">%s</a></li>\n""" % \ - (feed["feed_id"], feed["feed_title"]) - - posts = HTML_HEADER(feed["feed_title"], "../style.css") - posts += """<h1>Articles of the feed <a href="%s">%s</a></h1>\n""" % (feed["site_link"], feed["feed_title"]) - posts += """<p>%s articles.</p>\n""" % (format(mongo_db.nb_articles(feed["feed_id"]), ",d"),) - - for article in mongo_db.get_articles(feed_id=feed["feed_id"]): - - post_file_name = os.path.normpath(feed_folder + "/" + article["article_id"] + ".html") - feed_index = os.path.normpath(feed_folder + "/index.html") - - posts += article["article_date"].ctime() + " - " + \ - """<a href="./%s.html">%s</a>""" % \ - (article["article_id"], article["article_title"][:150]) + "<br />\n" - - a_post = HTML_HEADER(article["article_title"], "../style.css") - a_post += '<div style="width:60%; overflow:hidden; text-align:justify; margin:0 auto">\n' - a_post += """<h1><a href="%s">%s</a></h1>\n<br />""" % \ - (article["article_link"], article["article_title"]) - a_post += article["article_content"] - a_post += "</div>\n<hr />\n" - a_post += """<br />\n<a href="%s">Complete story</a>\n<br />\n""" % (article["article_link"],) - a_post += HTML_FOOTER - - with open(post_file_name, "w") as f: - f.write(a_post) - - posts += HTML_FOOTER - with open(feed_index, "w") as f: - f.write(posts) - - index += "</ul>\n" - index += "<p>" + time.strftime("Generated on %d %b %Y at %H:%M.") + "</p>\n" - index += HTML_FOOTER - with open(conf.path + "/var/export/webzine/" + "index.html", "w") as f: - f.write(index) - with open(conf.path + "/var/export/webzine/" + "style.css", "w") as f: - f.write(CSS) - -def export_txt(mongo_db): - """ - Export the articles given in parameter in text files. - """ - feeds = mongo_db.get_all_feeds() - for feed in feeds: - # creates folder for each stream - folder = conf.path + "/var/export/txt/" + \ - utils.normalize_filename(feed["feed_title"].strip().replace(':', '').lower()) - try: - os.makedirs(folder) - except OSError: - # directories already exists (not a problem) - pass - - for article in mongo_db.get_articles(feed_id=feed["feed_id"]): - name = article["article_date"].ctime().strip().replace(' ', '_') - name = os.path.normpath(folder + "/" + name + ".txt") - - content = "Title: " + article["article_title"] + "\n\n\n" - content += utils.clear_string(article["article_content"]) - - with open(name, "w") as f: - f.write(content) - -def export_epub(mongo_db): - """ - Export the articles given in parameter in ePub files. - """ - from epub import ez_epub - feeds = mongo_db.get_all_feeds() - for feed in feeds: - # creates folder for each stream - folder = conf.path + "/var/export/epub/" + \ - utils.normalize_filename(feed["feed_title"].strip().replace(':', '').lower().encode('utf-8')) - try: - os.makedirs(folder) - except OSError: - # directories already exists (not a problem) - pass - - for article in mongo_db.get_articles(feed_id=feed["feed_id"]): - name = article["article_date"].ctime().strip().replace(' ', '_') - name = os.path.normpath(folder + "/" + name + ".epub") - - section = ez_epub.Section() - section.title = article["article_title"] - section.paragraphs = [utils.clear_string(article["article_content"])] - ez_epub.makeBook(article["article_title"], [feed["feed_title"]], [section], \ - name, lang='en-US', cover=None) - -def export_pdf(feeds): - """ - Export the articles given in parameter in PDF files. - """ - from xhtml2pdf import pisa - import io as StringIO - for feed in list(feeds.values()): - # creates folder for each stream - folder = utils.path + "/var/export/pdf/" + \ - utils.normalize_filename(feed.feed_title.strip().replace(':', '').lower()) - try: - os.makedirs(folder) - except OSError: - # directories already exists (not a problem) - pass - - for article in list(feed.articles.values()): - name = article.article_date.strip().replace(' ', '_') - name = os.path.normpath(folder + "/" + name + ".pdf") - - content = HTML_HEADER(article.article_title) - content += '\n<div style="width: 50%; overflow:hidden; text-align: justify; margin:0 auto">\n' - content += """<h1><a href="%s">%s</a></h1><br />""" % \ - (article.article_link, article.article_title) - content += article.article_description - content += "</div>\n<hr />\n" - content += HTML_FOOTER - - try: - pdf = pisa.CreatePDF(StringIO.StringIO(content), file(name, "wb")) - except: - pass diff --git a/source/feedgetter.py b/source/feedgetter.py deleted file mode 100755 index ce1cba1b..00000000 --- a/source/feedgetter.py +++ /dev/null @@ -1,231 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 1.8 $" -__date__ = "$Date: 2010/09/02 $" -__revision__ = "$Date: 2013/08/15 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import hashlib -import threading -import urllib.request -import feedparser -from bs4 import BeautifulSoup -from datetime import datetime -from contextlib import contextmanager - -import conf -import search -import utils -import mongodb - -import log -pyaggr3g470r_log = log.Log() - -list_of_threads = [] - -@contextmanager -def opened_w_error(filename, mode="r"): - try: - f = open(filename, mode) - except IOError as err: - yield None, err - else: - try: - yield f, None - finally: - f.close() - -class FeedGetter(object): - """ - This class is in charge of retrieving feeds listed in ./var/feed.lst. - This class uses feedparser module from Mark Pilgrim. - For each feed a new thread is launched. - """ - def __init__(self): - """ - Initializes the database connection. - """ - # MongoDB connections - self.articles = mongodb.Articles(conf.MONGODB_ADDRESS, conf.MONGODB_PORT, \ - conf.MONGODB_DBNAME, conf.MONGODB_USER, conf.MONGODB_PASSWORD) - if conf.HTTP_PROXY == "": - self.proxy = urllib.request.ProxyHandler({}) - else: - self.proxy = urllib.request.ProxyHandler({"http" : conf.HTTP_PROXY}) - feedparser.USER_AGENT = conf.USER_AGENT - - def retrieve_feed(self, feed_url=None, feed_original=None): - """ - Parse the file 'feeds.lst' and launch a thread for each RSS feed. - """ - if feed_url != None: - self.process(feed_url, feed_original) - else: - with opened_w_error(conf.FEED_LIST) as (f, err): - if err: - pyaggr3g470r_log.error("List of feeds not found.") - else: - for a_feed in f: - # test if the URL is well formed - for url_regexp in utils.url_finders: - if url_regexp.match(a_feed): - the_good_url = url_regexp.match(a_feed).group(0).replace("\n", "") - try: - # launch a new thread for the RSS feed - thread = threading.Thread(None, self.process, \ - None, (the_good_url,)) - thread.start() - list_of_threads.append(thread) - except: - pass - break - - # wait for all threads are done - for th in list_of_threads: - th.join() - - def process(self, the_good_url, feed_original=None): - """Request the URL - - Executed in a thread. - """ - if utils.open_url(the_good_url)[0] == True: - # if ressource is available add the articles in the base. - self.add_into_database(the_good_url, feed_original) - - def add_into_database(self, feed_link, feed_original=None): - """ - Add the articles of the feed 'a_feed' in the database. - """ - a_feed = feedparser.parse(feed_link, handlers = [self.proxy]) - if a_feed['entries'] == []: - return - try: - feed_image = a_feed.feed.image.href - except: - feed_image = "/img/feed-icon-28x28.png" - - if feed_original != None: - feed_link = feed_original - - sha1_hash = hashlib.sha1() - sha1_hash.update(feed_link.encode('utf-8')) - feed_id = sha1_hash.hexdigest() - - feed = self.articles.get_feed(feed_id) - if None == feed: - collection_dic = {"feed_id": feed_id, \ - "type": 0, \ - "feed_image": feed_image, \ - "feed_title": utils.clear_string(a_feed.feed.title), \ - "feed_link": feed_link, \ - "site_link": a_feed.feed.link, \ - "mail": False \ - } - self.articles.add_collection(collection_dic) - feed = self.articles.get_feed(feed_id) - - articles = [] - for article in a_feed['entries']: - description = "" - article_title = "" - try: - # article content - description = article.content[0].value - except AttributeError: - try: - # article description - description = article.description - except Exception: - description = "" - try: - description = BeautifulSoup(description, "html.parser").decode() - article_title = BeautifulSoup(article.title, "html.parser").decode() - except Exception as E: - pyaggr3g470r_log.error("Problem when sanitizing the content of the feed: " + feed_link) - article_title = article.title - - try: - post_date = datetime(*article.published_parsed[:6]) - except: - post_date = datetime(*article.updated_parsed[:6]) - - sha1_hash = hashlib.sha1() - sha1_hash.update(article.link.encode('utf-8')) - article_id = sha1_hash.hexdigest() - - article = {"article_id": article_id, \ - "type":1, \ - "article_date": post_date, \ - "article_link": article.link, \ - "article_title": article_title, \ - "article_content": description, \ - "article_readed": False, \ - "article_like": False \ - } - - articles.append(article) - - if self.articles.get_articles(feed_id, article_id) == []: - # add the article to the Whoosh index - try: - search.add_to_index([article], feed) - except: - print("Whoosh error.") - pyaggr3g470r_log.error("Whoosh error.") - continue - - if conf.MAIL_ENABLED and feed["mail"]: - # if subscribed to the feed - threading.Thread(None, utils.send_mail, None, (conf.mail_from, conf.mail_to, \ - a_feed.feed.title, \ - article_title, description)).start() - self.articles.add_articles(articles, feed_id) - - -if __name__ == "__main__": - # Point of entry in execution mode - feed_getter = FeedGetter() - # Retrieve all feeds - feed_getter.retrieve_feed() - - # If you want to get all articles of a blog: - """ - for i in range(1,86): - feed_original = "http://esr.ibiblio.org/?feed=rss2" - feed = feed_original + "&paged=" + str(i) - print("Retrieving", feed, "...") - feed_getter.retrieve_feed(feed, feed_original) - """ - """ - for i in range(1,5): - feed_original = "http://spaf.wordpress.com/feed/" - feed = feed_original + "?paged=" + str(i) - print("Retrieving", feed, "...") - feed_getter.retrieve_feed(feed, feed_original) - """ - - # For a blogspot blog: - #feed_getter.retrieve_feed("http://www.blogger.com/feeds/4195135246107166251/posts/default", "http://neopythonic.blogspot.com/feeds/posts/default") - #feed_getter.retrieve_feed("http://www.blogger.com/feeds/8699431508730375743/posts/default", "http://python-history.blogspot.com/feeds/posts/default")
\ No newline at end of file diff --git a/source/log.py b/source/log.py deleted file mode 100755 index 5db5d838..00000000 --- a/source/log.py +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.1 $" -__date__ = "$Date: 2012/10/12 $" -__revision__ = "$Date: 2012/10/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -class Log(object): - """ - Log events. Especially events relative to authentication. - """ - def __init__(self): - """ - Initialization of the logger. - """ - import logging - self.logger = logging.getLogger("pyaggr3g470r") - hdlr = logging.FileHandler('./var/pyaggr3g470r.log') - formater = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - hdlr.setFormatter(formater) - self.logger.addHandler(hdlr) - self.logger.setLevel(logging.INFO) - - def info(self, message): - """ - Log notices. - """ - self.logger.info(message) - - def warning(self, message): - """ - Log warnings. - """ - self.logger.warning(message) - - def error(self, message): - """ - Log errors. - """ - self.logger.warning(message) - - def critical(self, message): - """ - Log critical errors. - """ - self.logger.critical(message) diff --git a/source/mongodb.py b/source/mongodb.py deleted file mode 100644 index 04cd44fa..00000000 --- a/source/mongodb.py +++ /dev/null @@ -1,283 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.8 $" -__date__ = "$Date: 2012/03/03 $" -__revision__ = "$Date: 2013/06/25 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import pymongo - -class Articles(object): - """ - This class is responsible of the management of the MongoDB - database. - """ - def __init__(self, url='localhost', port=27017, db_name="pyaggr3g470r", user="", password=""): - """ - Instantiates the connection. - """ - self.db_name = db_name - self.connection = pymongo.connection.Connection(url, port) - self.db = pymongo.database.Database(self.connection, self.db_name) - if password != "": - self.db.authenticate(user, password) - collections = self.db.collection_names() - for collection_name in collections: - if collection_name != "system.indexes": - self.db[collection_name].ensure_index([("article_date", pymongo.DESCENDING)], \ - name="date_index", unique=False, \ - background=True) - - def add_collection(self, new_collection): - """ - Creates a new collection for a new feed. - """ - collection = self.db[new_collection["feed_id"]] - collection.insert(new_collection) - - def add_articles(self, articles, feed_id): - """ - Add article(s) in a collection. - """ - collection = self.db[str(feed_id)] - for article in articles: - cursor = collection.find({"article_id":article["article_id"]}) - if cursor.count() == 0: - collection.insert(article) - - def delete_feed(self, feed_id): - """ - Delete a collection (feed with all articles). - """ - self.db.drop_collection(feed_id) - - def delete_article(self, feed_id, article_id): - """ - Delete an article. - """ - collection = self.db[str(feed_id)] - collection.remove(spec_or_id={"article_id":article_id}, safe=True) - - def get_feed(self, feed_id): - """ - Return information about a feed (collection). - Return None if the collection does not exist. - """ - try: - return next(self.db[str(feed_id)].find()) - except: - return None - - def get_all_feeds(self, condition=None): - """ - Return all feeds object. The returned list - is sorted by alphabetically (by feed name). - """ - feeds = [] - collections = self.db.collection_names() - for collection_name in collections: - if collection_name != "system.indexes": - if condition is None: - cursor = self.db[collection_name].find({"type":0}) - else: - cursor = self.db[collection_name].find({"type":0, condition[0]:condition[1]}) - if cursor.count() != 0: - feeds.append(next(cursor)) - feeds.sort(key = lambda elem: elem['feed_title'].lower()) - return feeds - - def get_articles(self, feed_id=None, article_id=None, condition=None, limit=1000000000): - """ - Return one or several articles. - The parameter "condition" is an optional requirement, for example: - get_articles(feed_id, condition=("article_readed", False)) will - return all unread articles of the feed 'feed_id'. - """ - if feed_id == None and article_id == None: - # Return all articles. - articles = [] - collections = self.db.collection_names() - for collection_name in collections: - collection = self.db[collection_name] - if condition is None: - articles.extend(collection.find({"type":1}, limit=limit)) - else: - articles.extend(collection.find({"type":1, condition[0]:condition[1]}, limit=limit)) - return articles - - elif feed_id != None and article_id == None: - # Return all the articles of a collection. - collection = self.db[str(feed_id)] - if condition is None: - cursor = collection.find({"type":1}, limit=limit) - else: - cursor = collection.find({"type":1, condition[0]:condition[1]}, limit=limit) - return cursor.sort([("article_date", pymongo.DESCENDING)]) - - elif feed_id != None and article_id != None: - # Return a precise article. - collection = self.db[str(feed_id)] - try: - return next(collection.find({"article_id":article_id})) - except: - return [] - - def get_favorites(self, feed_id=None): - """ - Return favorites articles. - """ - if feed_id is not None: - # only for a feed - collection = self.db[feed_id] - cursor = collection.find({'type':1, 'article_like':True}) - return cursor.sort([("article_date", pymongo.DESCENDING)]) - else: - favorites = [] - for feed_id in self.db.collection_names(): - favorites += self.get_favorites(feed_id) - return favorites - - def nb_articles(self, feed_id=None): - """ - Return the number of articles of a feed - or of all the database. - """ - if feed_id is not None: - collection = self.db[feed_id] - cursor = collection.find({'type':1}) - return cursor.count() - else: - nb_articles = 0 - for feed_id in self.db.collection_names(): - nb_articles += self.nb_articles(feed_id) - return nb_articles - - def nb_unread_articles(self, feed_id=None): - """ - Return the number of unread articles of a feed - or of all the database. - """ - if feed_id is not None: - return self.get_articles(feed_id=feed_id, condition=("article_readed", False)).count() - else: - return len(self.get_articles(condition=("article_readed", False))) - - def like_article(self, like, feed_id, article_id): - """ - Like or unlike an article. - """ - collection = self.db[str(feed_id)] - collection.update({"article_id": article_id}, {"$set": {"article_like": like}}) - - def nb_favorites(self, feed_id=None): - """ - Return the number of favorites articles of a feed - or of all the database. - """ - if feed_id is not None: - return self.get_favorites(feed_id).count() - else: - return len(self.get_favorites()) - - def nb_mail_notifications(self): - """ - Return the number of subscribed feeds. - """ - nb_mail_notifications = 0 - for feed_id in self.db.collection_names(): - collection = self.db[feed_id] - cursor = collection.find({'type':0, 'mail':True}) - nb_mail_notifications += cursor.count() - return nb_mail_notifications - - def mark_as_read(self, readed, feed_id=None, article_id=None): - """ - Mark one or several articles as read. - """ - if feed_id != None and article_id != None: - collection = self.db[str(feed_id)] - collection.update({"article_id": article_id, "article_readed":not readed}, {"$set": {"article_readed": readed}}) - elif feed_id != None and article_id == None: - collection = self.db[str(feed_id)] - collection.update({"type": 1, "article_readed":not readed}, {"$set": {"article_readed": readed}}, multi=True) - else: - for feed_id in self.db.collection_names(): - self.mark_as_read(readed, feed_id, None) - - def update_feed(self, feed_id, changes): - """ - Update a feed. - """ - collection = self.db[str(feed_id)] - collection.update({"type": 0, "feed_id":feed_id}, {"$set": changes}, multi=True) - if "feed_id" in changes.keys(): - self.db[str(feed_id)].rename(str(changes["feed_id"])) - - # Functions on database - def drop_database(self): - """ - Drop all the database - """ - self.connection.drop_database(self.db_name) - - -if __name__ == "__main__": - # Point of entry in execution mode. - articles = Articles() - # Create a collection for a stream - collection_dic = {"collection_id": 42,\ - "feed_image": "Image", \ - "feed_title": "Title", \ - "feed_link": "Link", \ - "site_title": "Site link", \ - "mail": True, \ - } - #articles.add_collection(collection_dic) - - # Add an article in the newly created collection - article_dic1 = {"article_id": 51, \ - "article_date": "Today", \ - "article_link": "Link of the article", \ - "article_title": "The title", \ - "article_content": "The content of the article", \ - "article_readed": True, \ - "article_like": True \ - } - article_dic2 = {"article_id": 52, \ - "article_date": "Yesterday", \ - "article_link": "Link", \ - "article_title": "Hello", \ - "article_content": "The content of the article", \ - "article_readed": True, \ - "article_like": True \ - } - - #articles.add_articles([article_dic1, article_dic2], 42) - - print("All articles:") - #print articles.get_all_articles() - - - # Drop the database - #articles.drop_database() diff --git a/source/pyAggr3g470r b/source/pyAggr3g470r deleted file mode 100755 index 3755ad16..00000000 --- a/source/pyAggr3g470r +++ /dev/null @@ -1,143 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2012 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2011/06/20 $" -__date__ = "$Date: 2013/02/17 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -# This control file is inspired from Forban: http://www.foo.be/forban. - -import os -import sys -import time -import subprocess -import platform -import signal - -PATH = os.path.abspath(".") -SERVICE = "pyAggr3g470r" - -def service_start(python_command, servicename=None): - """ - Starts a new service with Popen and returns the processus id. - """ - if servicename is not None: - service = servicename + ".py" - proc = subprocess.Popen([python_command, "-tt", service], stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - time.sleep(0.15) - return proc.pid - return False - -def writepid(processname=None, pid=None): - """ - Writes the pid of processname in a file. - """ - pidpath = os.path.join(PATH, "var", processname + ".pid") - if processname is not None and pid is not None: - with open(pidpath, "w") as f: - f.write(str(pid)) - return True - return False - -def checkpid(servicename=None): - pidpath = os.path.join(PATH,"var", servicename + ".pid") - if os.path.exists(pidpath): - return True - else: - return False - -def pidof(processname=None): - pidpath = os.path.join(PATH, "var", processname + ".pid") - if processname is not None and os.path.exists(pidpath): - with open(pidpath) as f: - pid = f.read() - return pid - return False - -def rmpid(processname=None): - """ - Deletes the file which contains the PID. - """ - pidpath = os.path.join(PATH, "var", processname + ".pid") - if os.path.exists(pidpath): - os.unlink(pidpath) - return True - else: - return False - -def start(python_command): - if not checkpid(servicename=SERVICE): - pid = service_start(python_command, servicename =SERVICE) - writepid(processname=SERVICE, pid=pid) - print(SERVICE + " is starting with pid: " + pidof(processname=SERVICE)) - else: - print(SERVICE + " could not be started (pid exists)") - retval = False - -def stop(): - """ - Stop the process SERVICE. - """ - print("Stopping " + SERVICE + "...") - retval = True - pid = pidof(processname=SERVICE) - if pid: - if platform.system() == "Windows": - import win32api - import win32con - phandle = win32api.OpenProcess(win32con.PROCESS_TERMINATE, 0, int(pid)) - win32api.TerminateProcess(phandle, 0) - win32api.CloseHandle(phandle) - rmpid(processname=SERVICE) - else: - try: - os.kill(int(pid), signal.SIGKILL) - except OSError as e: - print(SERVICE + " unsuccessfully stopped") - retval = False - finally: - rmpid(processname=SERVICE) - return retval - -def usage(): - print("pyAggr3g470r (start|stop|restart)") - exit (1) - -if __name__ == "__main__": - # Point of entry in execution mode. - python_command = "python" - if sys.version_info.major == 2: - # Ensures that code doesn't illegally mixed tabs and spaces - python_command = "python3.3" - if len(sys.argv) == 1: - usage() - elif sys.argv[1] == "start": - start(python_command) - elif sys.argv[1] == "stop": - stop() - elif sys.argv[1] == "restart": - stop() - start(python_command) - else: - usage() diff --git a/source/pyAggr3g470r.py b/source/pyAggr3g470r.py deleted file mode 100755 index 922e7114..00000000 --- a/source/pyAggr3g470r.py +++ /dev/null @@ -1,715 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 4.1 $" -__date__ = "$Date: 2010/01/29 $" -__revision__ = "$Date: 2013/09/09 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -# -# This file contains the "Root" class which describes -# all pages (views) of pyAggr3g470r. These pages are: -# - main page; -# - management; -# - history; -# - favorites; -# - notifications; -# - unread; -# - feed summary; -# - inactives; -# - languages. -# Templates are described in ./templates with the Mako -# template library. -# - -import os -import re -import time -import datetime - -from collections import defaultdict -from whoosh.index import EmptyIndexError - -import cherrypy -from mako.template import Template -from mako.lookup import TemplateLookup -lookup = TemplateLookup(directories=['templates']) - -import conf -import utils -import export -import mongodb -import search -import feedgetter -import auth - -def error_404(status, message, traceback, version): - """ - Display an error if the page does not exist. - """ - message = "<p>Error %s - This page does not exist.</p>" % status - tmpl = lookup.get_template("error.html") - return tmpl.render(message=message) - -def handle_error(): - """ - Handle different type of errors. - """ - message = "<p>Sorry, an error occured.</p>" - cherrypy.response.status = 500 - cherrypy.response.body = [message] - -class RestrictedArea(object): - """ - All methods in this controller (and subcontrollers) is - open only to members of the admin group - """ - _cp_config = { - 'auth.auth.require': [auth.member_of('admin')] - } - - @cherrypy.expose - def index(self): - message = "<p>This is the admin only area.</p>" - tmpl = lookup.get_template("error.html") - return tmpl.render(message=message) - -class pyAggr3g470r(object): - """ - Main class. - All pages of pyAggr3g470r are described in this class. - """ - _cp_config = {'request.error_response': handle_error, \ - 'tools.sessions.on': True, \ - 'tools.auth.on': True} - - def __init__(self): - """ - """ - self.auth = auth.AuthController() - restricted = RestrictedArea() - - self.mongo = mongodb.Articles(conf.MONGODB_ADDRESS, conf.MONGODB_PORT, \ - conf.MONGODB_DBNAME, conf.MONGODB_USER, conf.MONGODB_PASSWORD) - @auth.require() - def index(self): - """ - Main page containing the list of feeds and articles. - """ - feeds = self.mongo.get_all_feeds() - nb_unread_articles = self.mongo.nb_unread_articles() - nb_favorites = self.mongo.nb_favorites() - nb_mail_notifications = self.mongo.nb_mail_notifications() - tmpl = lookup.get_template("index.html") - return tmpl.render(feeds=feeds, nb_feeds=len(feeds), mongo=self.mongo, \ - nb_favorites=nb_favorites, nb_unread_articles=nb_unread_articles, \ - nb_mail_notifications=nb_mail_notifications, header_text=nb_unread_articles) - - index.exposed = True - - @auth.require() - def management(self): - """ - Management page. - Allows adding and deleting feeds. Export functions of the MongoDB data base - and display some statistics. - """ - feeds = self.mongo.get_all_feeds() - nb_mail_notifications = self.mongo.nb_mail_notifications() - nb_favorites = self.mongo.nb_favorites() - nb_articles = format(self.mongo.nb_articles(), ",d") - nb_unread_articles = format(self.mongo.nb_unread_articles(), ",d") - nb_indexed_documents = format(search.nb_documents(), ",d") - tmpl = lookup.get_template("management.html") - return tmpl.render(feeds=feeds, nb_mail_notifications=nb_mail_notifications, \ - nb_favorites=nb_favorites, nb_articles=nb_articles, \ - nb_unread_articles=nb_unread_articles, \ - mail_notification_enabled=conf.MAIL_ENABLED, \ - nb_indexed_documents=nb_indexed_documents) - - management.exposed = True - - @auth.require() - def statistics(self, word_size=6): - """ - More advanced statistics. - """ - articles = self.mongo.get_articles() - top_words = utils.top_words(articles, n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) - tmpl = lookup.get_template("statistics.html") - return tmpl.render(articles=articles, word_size=word_size, tag_cloud=tag_cloud) - - statistics.exposed = True - - @auth.require() - def search(self, query=None): - """ - Simply search for the string 'query' - in the description of the article. - """ - param, _, value = query.partition(':') - feed_id = None - if param == "Feed": - feed_id, _, query = value.partition(':') - search_result = defaultdict(list) - try: - results = search.search(param) - except EmptyIndexError as e: - return self.error('<p>The database has not been <a href="/index_base">indexed</a>.</p>') - for result in results: - article = self.mongo.get_articles(result[0], result[1]) - if article != []: - search_result[result[0]].append(article) - sorted_search_result = {feed_id: sorted(articles, key=lambda t: t['article_date'], reverse=True) \ - for feed_id, articles in search_result.items()} - tmpl = lookup.get_template("search.html") - return tmpl.render(search_result=sorted_search_result, query=query, feed_id=feed_id, mongo=self.mongo) - - search.exposed = True - - @auth.require() - def fetch(self, param=None): - """ - Fetch all feeds. - """ - feed_link = None - if None != param: - # Fetch only the feed specified in parameter - feed_link = self.mongo.get_feed(param)["feed_link"] - feed_getter = feedgetter.FeedGetter() - feed_getter.retrieve_feed(feed_url=feed_link) - return self.index() - - fetch.exposed = True - - @auth.require() - def article(self, param, plain_text=0): - """ - Display the article in parameter in a new Web page. - """ - try: - feed_id, article_id = param.split(':') - article = self.mongo.get_articles(feed_id, article_id) - if article == []: - return self.error("<p>This article do not exists.</p>") - feed = self.mongo.get_feed(feed_id) - articles = self.mongo.get_articles(feed_id) - except: - return self.error("<p>Bad URL. This article do not exists.</p>") - - if article["article_readed"] == False: - # if the current article is not yet readed, update the database - self.mark_as_read("Article:"+article["article_id"]+":"+feed["feed_id"]) - - # Description (full content) of the article - if plain_text == "1": - description = "<p>" + utils.clear_string(article["article_content"]) + "</p>" - else: - description = article["article_content"] - if description == "": - description = "<p>No description available.</p>" - - # Generation of the QR Code for the current article - utils.generate_qr_code(article) - - # Previous and following articles - previous, following = None, None - liste = self.mongo.get_articles(feed_id) - for current_article in self.mongo.get_articles(feed_id): - next(articles) - if current_article["article_id"] == article_id: - break - following = current_article - if following is None: - following = liste[liste.count()-1] - try: - previous = next(articles) - except StopIteration: - previous = liste[0] - - tmpl = lookup.get_template("article.html") - return tmpl.render(header_text=article["article_title"], article=article, previous=previous, following=following, \ - diaspora=conf.DIASPORA_POD, feed=feed, description=description, plain_text=plain_text) - - article.exposed = True - - @auth.require() - def feed(self, feed_id, word_size=6): - """ - This page gives summary informations about a feed (number of articles, - unread articles, average activity, tag cloud, e-mail notification and - favourite articles for the current feed. - """ - try: - feed = self.mongo.get_feed(feed_id) - if feed == None: - return self.error("<p>This feed do not exists.</p>") - articles = self.mongo.get_articles(feed_id, limit=10) - nb_articles_feed = self.mongo.nb_articles(feed_id) - nb_articles_total = self.mongo.nb_articles() - nb_unread_articles_feed = self.mongo.nb_unread_articles(feed_id) - favorites = self.mongo.get_favorites(feed_id) - nb_favorites = self.mongo.nb_favorites(feed_id) - except KeyError: - return self.error("<p>This feed do not exists.</p>") - - - if articles.count() != 0: - today = datetime.datetime.now() - last_article = articles[0]["article_date"] - first_article = articles[self.mongo.nb_articles(feed_id)-2]["article_date"] - delta = last_article - first_article - elapsed = today - last_article - average = round(nb_articles_feed / abs(delta.days), 2) - - top_words = utils.top_words(articles = self.mongo.get_articles(feed_id), n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) - - tmpl = lookup.get_template("feed.html") - return tmpl.render(feed=feed, articles=articles, favorites=favorites, \ - nb_articles_feed=nb_articles_feed, nb_articles_total=nb_articles_total, nb_unread_articles_feed=nb_unread_articles_feed, \ - nb_favorites = nb_favorites, first_post_date=first_article, end_post_date=last_article, \ - average=average, delta=delta, elapsed=elapsed, \ - tag_cloud=tag_cloud, word_size=word_size, \ - mail_to=conf.mail_to, mail_notification_enabled=conf.MAIL_ENABLED) - - tmpl = lookup.get_template("feed.html") - return tmpl.render(feed=feed, articles=[]) - - feed.exposed = True - - @auth.require() - def articles(self, feed_id): - """ - This page displays all articles of a feed. - """ - try: - feed = self.mongo.get_feed(feed_id) - articles = self.mongo.get_articles(feed_id) - except KeyError: - return self.error("<p>This feed do not exists.</p>") - tmpl = lookup.get_template("articles.html") - return tmpl.render(articles=articles, feed=feed) - - articles.exposed = True - - @auth.require() - def unread(self, feed_id=""): - """ - This page displays all unread articles of a feed. - """ - feeds = self.mongo.get_all_feeds() - tmpl = lookup.get_template("unread.html") - return tmpl.render(feeds=feeds, feed_id=feed_id, mongo=self.mongo) - unread.exposed = True - - @auth.require() - def history(self, query="all", m=""): - """ - This page enables to browse articles chronologically. - """ - feeds = self.mongo.get_all_feeds() - tmpl = lookup.get_template("history.html") - return tmpl.render(feeds=feeds, mongo=self.mongo, query=query, m=m) - - history.exposed = True - - @auth.require() - def error(self, message): - """ - Display a message (bad feed id, bad article id, etc.) - """ - tmpl = lookup.get_template("error.html") - return tmpl.render(message=message) - - error.exposed = True - - @auth.require() - def mark_as_read(self, target=""): - """ - Mark one (or more) article(s) as read by setting the value of the field - 'article_readed' of the MongoDB database to 'True'. - """ - param, _, identifiant = target.partition(':') - - # Mark all articles as read. - if param == "": - self.mongo.mark_as_read(True, None, None) - # Mark all articles from a feed as read. - elif param == "Feed" or param == "Feed_FromMainPage": - self.mongo.mark_as_read(True, identifiant, None) - # Mark an article as read. - elif param == "Article": - self.mongo.mark_as_read(True, identifiant.split(':')[1], identifiant.split(':')[0]) - return self.index() - - mark_as_read.exposed = True - - @auth.require() - def notifications(self): - """ - List all active e-mail notifications. - """ - feeds = self.mongo.get_all_feeds(condition=("mail",True)) - tmpl = lookup.get_template("notifications.html") - return tmpl.render(feeds=feeds, mail_to=conf.mail_to, mail_notification_enabled=conf.MAIL_ENABLED) - - notifications.exposed = True - - @auth.require() - def mail_notification(self, param): - """ - Enable or disable to notifications of news for a feed. - """ - try: - action, feed_id = param.split(':') - new_value = 1 == int(action) - self.mongo.update_feed(feed_id, {"mail":new_value}) - except: - return self.error("<p>Bad URL. This feed do not exists.</p>") - return self.index() - - mail_notification.exposed = True - - @auth.require() - def like(self, param): - """ - Mark or unmark an article as favorites. - """ - try: - like, feed_id, article_id = param.split(':') - articles = self.mongo.get_articles(feed_id, article_id) - except: - return self.error("<p>Bad URL. This article do not exists.</p>") - self.mongo.like_article("1"==like, feed_id, article_id) - return self.article(feed_id+":"+article_id) - - like.exposed = True - - @auth.require() - def subscriptions(self): - """ - List all active e-mail notifications. - """ - feeds = self.mongo.get_all_feeds() - tmpl = lookup.get_template("subscriptions.html") - return tmpl.render(feeds=feeds) - - subscriptions.exposed = True - - @auth.require() - def favorites(self): - """ - List of favorites articles - """ - feeds = self.mongo.get_all_feeds() - articles = {} - for feed in feeds: - articles[feed["feed_id"]] = self.mongo.get_favorites(feed["feed_id"]) - tmpl = lookup.get_template("favorites.html") - return tmpl.render(feeds=feeds, \ - articles=articles) - - favorites.exposed = True - - @auth.require() - def inactives(self, nb_days=365): - """ - List of favorites articles - """ - feeds = self.mongo.get_all_feeds() - today = datetime.datetime.now() - inactives = [] - for feed in feeds: - more_recent_article = self.mongo.get_articles(feed["feed_id"], limit=1) - if more_recent_article.count() == 0: - last_post = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0))) - else: - last_post = next(more_recent_article)["article_date"] - elapsed = today - last_post - if elapsed > datetime.timedelta(days=int(nb_days)): - inactives.append((feed, elapsed)) - tmpl = lookup.get_template("inactives.html") - return tmpl.render(inactives=inactives, nb_days=int(nb_days)) - - inactives.exposed = True - - @auth.require() - def languages(self): - """ - Filter by languages. - """ - try: - from guess_language import guess_language_name - except: - tmpl = lookup.get_template("error.html") - return tmpl.render(message='<p>Module <i><a href="https://bitbucket.org/spirit/guess_language/">guess_language</a></i> not installed.</p>') - result = {} - feeds = self.mongo.get_all_feeds() - for feed in feeds: - for article in self.mongo.get_articles(feed["feed_id"]): - language = guess_language_name(utils.clear_string(article["article_content"])) - result.setdefault(language, defaultdict(list)) - result[language][feed["feed_id"]].append(article) - tmpl = lookup.get_template("languages.html") - return tmpl.render(articles_sorted_by_languages=result, mongo=self.mongo) - - languages.exposed = True - - @auth.require() - def add_feed(self, url): - """ - Add a new feed with the URL of a page. - """ - # search the feed in the HTML page with BeautifulSoup - feed_url = utils.search_feed(url) - if feed_url is None: - return self.error("<p>Impossible to find a feed at this URL.</p>") - # if a feed exists - else: - result = utils.add_feed(feed_url) - # if the feed is not in the file feed.lst - import hashlib - sha1_hash = hashlib.sha1() - sha1_hash.update(feed_url.encode('utf-8')) - feed_id = sha1_hash.hexdigest() - if result is False: - message = """<p>You are already following <a href="/feed/%s">this feed</a>!</p>""" % (feed_id,) - else: - message = """<p><a href="/feed/%s">Feed added</a>. You can now <a href="/fetch/">fetch your feeds</a>.</p>""" % (feed_id,) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message=message) - - add_feed.exposed = True - - @auth.require() - def remove_feed(self, feed_id): - """ - Remove a feed from the file feed.lst and from the MongoDB database. - """ - feed = self.mongo.get_feed(feed_id) - self.mongo.delete_feed(feed_id) - utils.remove_feed(feed["feed_link"]) - message = """<p>All articles from the feed <i>%s</i> are now removed from the base.</p>""" % (feed["feed_title"],) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message=message) - - remove_feed.exposed = True - - @auth.require() - def change_site_url(self, feed_id, old_site_url, new_site_url): - """ - Enables to change the URL of a site present in the database. - """ - try: - self.mongo.update_feed(feed_id, {"site_link":new_site_url}) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>The URL of the site has been changed.</p>") - except: - return self.error("<p>Error when changing the URL of the site.</p>") - - change_site_url.exposed = True - - @auth.require() - def change_feed_url(self, feed_id, old_feed_url, new_feed_url): - """ - Enables to change the URL of a feed already present in the database. - """ - import hashlib - sha1_hash = hashlib.sha1() - sha1_hash.update(new_feed_url.encode('utf-8')) - new_feed_id = sha1_hash.hexdigest() - self.mongo.update_feed(feed_id, {"feed_id":new_feed_id, - "feed_link":new_feed_url}) - result = utils.change_feed_url(old_feed_url, new_feed_url) - if result: - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>The URL of the feed has been changed.</p>") - else: - return self.error("<p>Error when changing the URL of the feed.</p>") - - change_feed_url.exposed = True - - @auth.require() - def change_feed_name(self, feed_id, new_feed_name): - """ - Enables to change the name of a feed. - """ - try: - self.mongo.update_feed(feed_id, {"feed_title":new_feed_name}) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>The name of the feed has been changed.</p>") - except: - return self.error("<p>Error when changing the name of the feed.</p>") - - change_feed_name.exposed = True - - @auth.require() - def change_feed_logo(self, feed_id, new_feed_logo): - """ - Enables to change the name of a feed. - """ - try: - self.mongo.update_feed(feed_id, {"feed_image":new_feed_logo}) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>The logo of the feed has been changed.</p>") - except: - return self.error("<p>Error when changing the logo of the feed.</p>") - - change_feed_logo.exposed = True - - @auth.require() - def change_username(self, new_username): - """ - Enables to change the username of a user. - """ - result = auth.change_username(self.auth.username, new_username) - if result: - self.auth.username = new_username - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>Your username has been changed.</p>") - else: - return self.error("<p>Impossible to change the username.</p>") - - change_username.exposed = True - - @auth.require() - def change_password(self, new_password): - """ - Enables to change the password of a user. - """ - result = auth.change_password(self.auth.username, new_password) - if result: - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>Your password has been changed.</p>") - else: - return self.error("<p>Impossible to change the password.</p>") - - change_password.exposed = True - - @auth.require() - def delete_article(self, param): - """ - Delete an article. - """ - try: - feed_id, article_id = param.split(':') - # Delete from the MonfoDB database - self.mongo.delete_article(feed_id, article_id) - # Delete from the Whoosh index - search.delete_article(feed_id, article_id) - except: - return self.error("<p>Bad URL. This article do not exists.</p>") - - return self.index() - - delete_article.exposed = True - - @auth.require() - def logout(self): - """ - Close the session. - """ - return self.auth.logout() - - logout.exposed = True - - @auth.require() - def drop_base(self): - """ - Delete all articles. - """ - self.mongo.drop_database() - return self.index() - - drop_base.exposed = True - - @auth.require() - def index_base(self): - """ - Launches the indexing of the database. - """ - search.create_index() - return self.index() - - index_base.exposed = True - - @auth.require() - def export(self, export_method): - """ - Export articles currently loaded from the MongoDB database with - the appropriate function of the 'export' module. - """ - getattr(export, export_method)(self.mongo) - try: - getattr(export, export_method)(self.mongo) - except Exception as e: - return self.error(e) - tmpl = lookup.get_template("confirmation.html") - return tmpl.render(message="<p>Export successfully terminated.<br />Check the folder: <b>" + conf.path + "/var/export/</b>.</p>") - - export.exposed = True - - @auth.require() - def epub(self, param): - """ - Export an article to EPUB. - """ - try: - from epub import ez_epub - except Exception as e: - return self.error(e) - try: - feed_id, article_id = param.split(':') - except: - return self.error("Bad URL.") - try: - feed_id, article_id = param.split(':') - feed = self.mongo.get_feed(feed_id) - articles = self.mongo.get_articles(feed_id) - article = self.mongo.get_articles(feed_id, article_id) - except: - self.error("<p>This article do not exists.</p>") - try: - folder = conf.path + "/var/export/epub/" - os.makedirs(folder) - except OSError: - # directories already exists (not a problem) - pass - section = ez_epub.Section() - section.title = article["article_title"] - section.paragraphs = [utils.clear_string(article["article_content"])] - ez_epub.makeBook(article["article_title"], [feed["feed_title"]], [section], \ - os.path.normpath(folder) + "article.epub", lang='en-US', cover=None) - return self.article(param) - - epub.exposed = True - - -if __name__ == '__main__': - # Point of entry in execution mode - root = pyAggr3g470r() - root.favicon_ico = cherrypy.tools.staticfile.handler(filename=os.path.join(conf.path + "/static/img/favicon.png")) - cherrypy.config.update({'error_page.404': error_404}) - cherrypy.quickstart(root, "/" ,config=conf.path + "/cfg/cherrypy.cfg") diff --git a/source/search.py b/source/search.py deleted file mode 100644 index a9248a09..00000000 --- a/source/search.py +++ /dev/null @@ -1,129 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2013/06/24 $" -__revision__ = "$Date: 2013/06/25 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import os - -from whoosh.index import create_in, open_dir -from whoosh.index import EmptyIndexError -from whoosh.fields import * -from whoosh.query import * -from whoosh.qparser import QueryParser -from whoosh.writing import AsyncWriter - -import conf -import utils -import mongodb - -indexdir = "./var/indexdir" - -schema = Schema(title=TEXT(stored=True), \ - content=TEXT, \ - article_id=TEXT(stored=True), \ - feed_id=TEXT(stored=True)) - -def create_index(): - """ - Creates the index. - """ - mongo = mongodb.Articles(conf.MONGODB_ADDRESS, conf.MONGODB_PORT, \ - conf.MONGODB_DBNAME, conf.MONGODB_USER, conf.MONGODB_PASSWORD) - feeds = mongo.get_all_feeds() - if not os.path.exists(indexdir): - os.mkdir(indexdir) - ix = create_in(indexdir, schema) - writer = ix.writer() - for feed in feeds: - for article in mongo.get_articles(feed["feed_id"]): - writer.add_document(title=article["article_title"], \ - content=utils.clear_string(article["article_content"]), \ - article_id=article["article_id"] , \ - feed_id=feed["feed_id"]) - writer.commit() - -def add_to_index(articles, feed): - """ - Add a list of articles to the index. - Here an AsyncWriter is used because the function will - be called in multiple threads by the feedgetter module. - """ - try: - ix = open_dir(indexdir) - except (EmptyIndexError, OSError) as e: - raise EmptyIndexError - writer = AsyncWriter(ix) - for article in articles: - writer.add_document(title=article["article_title"], \ - content=utils.clear_string(article["article_content"]), \ - article_id=article["article_id"] , \ - feed_id=feed["feed_id"]) - writer.commit() - -def delete_article(feed_id, article_id): - """ - Delete an article from the index. - """ - try: - ix = open_dir(indexdir) - except (EmptyIndexError, OSError) as e: - raise EmptyIndexError - writer = ix.writer() - document = And([Term("feed_id", feed_id), Term("article_id", article_id)]) - writer.delete_by_query(document) - writer.commit() - -def search(term): - """ - Search for `term` in the index. - Returns a list of articles. - """ - try: - ix = open_dir(indexdir) - except (EmptyIndexError, OSError) as e: - raise EmptyIndexError - with ix.searcher() as searcher: - query = QueryParser("content", ix.schema).parse(term) - results = searcher.search(query, limit=None) - return [(article["feed_id"], article["article_id"]) for article in results] - -def nb_documents(): - """ - Return the number of undeleted documents. - """ - try: - ix = open_dir(indexdir) - except (EmptyIndexError, OSError) as e: - raise EmptyIndexError - return ix.doc_count() - -if __name__ == "__main__": - # Point of entry in execution mode. - #create_index() - print(nb_documents()) - results = search("Nothomb") - for article in results: - print(article) diff --git a/source/static/css/log.css b/source/static/css/log.css deleted file mode 100644 index 56eb5220..00000000 --- a/source/static/css/log.css +++ /dev/null @@ -1,24 +0,0 @@ -html { - height: 100%; -} - -body { - font:normal medium 'Gill Sans','Gill Sans MT',Verdana,sans-serif; - line-height:1.75; - height: 100%; - margin: 0; - padding: 0; -} - -#logform { - text-align:center; - clear: both; - position: float; - vertical-align: center; - margin-top: 10%; - -} - -img { - border:0; -} diff --git a/source/static/css/style.css b/source/static/css/style.css deleted file mode 100755 index e99bfb7c..00000000 --- a/source/static/css/style.css +++ /dev/null @@ -1,222 +0,0 @@ -html, body { - margin: 0px 0px 0px 5px; - padding: 0px 0px 0px 0px; - height: 100%; - background-color: white; - color: black; - text-align: justify; - font: normal small 'Gill Sans','Gill Sans MT',Verdana,sans-serif; -} - -img { - border: 0px; -} - -h1 { - font-size: 100%; - margin: 0em 0em; - padding: 0px; -} - -h2 { - margin: 0.0em 0em; - padding: 0px; - font-style: normal; - font-variant: normal; - font-weight: bold; - font-size: 100%; - letter-spacing: 0em; - text-align: right; -} - -h3 { - margin: 0em 0em 0.5em 0em; - font-size: 100%; - font-weight: bold; - text-align: left; -} - -h1 a, h2 a, h3 a { - text-decoration: none; -} - -a:link, a:visited { - color: #003399; - text-decoration:none -} - -a:hover { - color: blue; -} - -hr { - color: white; - border-top: dotted black; - border-width: 1px 0px 0px 0px; - margin: 1em 0em; -} - -/* Menu */ -.menu_container { - position:fixed; - margin:0px; - padding:0px; - z-index:4; -} - -/* Navigation bars */ -.nav_container { - position:fixed; - right:5px; - margin:5px; - padding:5px; - white-space:nowrap; - z-index:3; - clear:both; - border-style:dashed; - border-width:thin; - border-color:#98bf21; -} -.nav_container.horizontal { - position:absolute; - white-space:normal; - z-index:4; - width:*; -} -.nav_container.horizontal div { - float:right; - padding-right:10px; -} - -#nav { - position: absolute; - top: 0px; - right: 0px; - z-index: 2; -} - -#heading, #nav ul, #nav li { - background: #EEEEEE; - color: inherit; - border-bottom: 0px solid black; -} - -#heading h1 { - margin: 0; - padding: 0.4em 0 0.4em 2em; - white-space: nowrap; -} - -#nav ul { - display: block; - margin: 0em 0em 0em 0em; - border-right: 0px solid #000000; -} - -#nav li { - margin: 0em; - padding: 0.1em 0em 0.2em 0.1em; - display: block; - float: left; - border-left: 0px solid #000000; -} - -.right { - clear: right; - float: right; - text-align: right; - margin: 0em 1em 0em 1em; - max-width: 25%; -} - -img.right { - width: auto; -} - -.inner .right { - margin-right: 0em; -} - -.right blockquote { - float: right; - position: relative; - clear: both; - padding-top: 1em; - padding-bottom: 0em; - margin-bottom: 0em; - font-style: italic; - width: 100%; - margin-right: 0em; -} - -blockquote.right { - width: 50%; - margin-left: 50%; - margin-right: 0em; -} - -/* Classes */ - -.clear { - font-size: 1px; - height: 0px; - clear: both; -} - -.invisible { - display: none; -} - -.inner { - margin-top: 0em; - padding: 0em 0em 0em 0em; - clear: both; -} - -.innerlogo { - margin-top: 0em; - padding: 0em 0em 0em 0em; - clear: both; -} - -.left { - float: left; - position: absolute; -} - -.tex { - position: relative; top: 0.2em; - margin-left: -0.2em; - margin-right: -0.1em; -} - -/* CSS ToolTips */ -.tooltip { - color: #FFF; - outline: none; - text-decoration: none; - position: relative; -} - -.tooltip span { - color: #FFF; - margin-left: -999em; - position: absolute; -} - -.tooltip:hover span { - border-radius: 5px 5px; -moz-border-radius: 5px; -webkit-border-radius: 5px; - box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.1); -webkit-box-shadow: 5px 5px rgba(0, 0, 0, 0.1); -moz-box-shadow: 5px 5px rgba(0, 0, 0, 0.1); - font-family: Calibri, Tahoma, Geneva, sans-serif; - position: absolute; left: 1em; top: 2em; z-index: 99; - margin-left: 0; width: 250px; -} -.classic { - padding: 0.8em 1em; - background: rgba(0, 0, 0, 0.85); - border: 5px 5px; -} - -* html a:hover { - background: transparent; -} diff --git a/source/static/img/blogmarks.png b/source/static/img/blogmarks.png Binary files differdeleted file mode 100755 index 2464e5bb..00000000 --- a/source/static/img/blogmarks.png +++ /dev/null diff --git a/source/static/img/check-news.png b/source/static/img/check-news.png Binary files differdeleted file mode 100644 index cce7df39..00000000 --- a/source/static/img/check-news.png +++ /dev/null diff --git a/source/static/img/cross.png b/source/static/img/cross.png Binary files differdeleted file mode 100644 index 1514d51a..00000000 --- a/source/static/img/cross.png +++ /dev/null diff --git a/source/static/img/diaspora.png b/source/static/img/diaspora.png Binary files differdeleted file mode 100644 index fdf8bb72..00000000 --- a/source/static/img/diaspora.png +++ /dev/null diff --git a/source/static/img/digg.png b/source/static/img/digg.png Binary files differdeleted file mode 100755 index 097c4600..00000000 --- a/source/static/img/digg.png +++ /dev/null diff --git a/source/static/img/email-follow.png b/source/static/img/email-follow.png Binary files differdeleted file mode 100644 index 4505c610..00000000 --- a/source/static/img/email-follow.png +++ /dev/null diff --git a/source/static/img/favicon.png b/source/static/img/favicon.png Binary files differdeleted file mode 100644 index d4d38473..00000000 --- a/source/static/img/favicon.png +++ /dev/null diff --git a/source/static/img/feed-icon-28x28.png b/source/static/img/feed-icon-28x28.png Binary files differdeleted file mode 100755 index d64c669c..00000000 --- a/source/static/img/feed-icon-28x28.png +++ /dev/null diff --git a/source/static/img/following-article.png b/source/static/img/following-article.png Binary files differdeleted file mode 100644 index 0e59e459..00000000 --- a/source/static/img/following-article.png +++ /dev/null diff --git a/source/static/img/hacker-news.png b/source/static/img/hacker-news.png Binary files differdeleted file mode 100644 index ce92765d..00000000 --- a/source/static/img/hacker-news.png +++ /dev/null diff --git a/source/static/img/heart-32x32.png b/source/static/img/heart-32x32.png Binary files differdeleted file mode 100644 index 09b01cb5..00000000 --- a/source/static/img/heart-32x32.png +++ /dev/null diff --git a/source/static/img/heart.png b/source/static/img/heart.png Binary files differdeleted file mode 100644 index f36f3cfd..00000000 --- a/source/static/img/heart.png +++ /dev/null diff --git a/source/static/img/heart_open.png b/source/static/img/heart_open.png Binary files differdeleted file mode 100644 index e1c6e027..00000000 --- a/source/static/img/heart_open.png +++ /dev/null diff --git a/source/static/img/history.png b/source/static/img/history.png Binary files differdeleted file mode 100644 index 2a57cc17..00000000 --- a/source/static/img/history.png +++ /dev/null diff --git a/source/static/img/identica.png b/source/static/img/identica.png Binary files differdeleted file mode 100644 index 18b5bd2b..00000000 --- a/source/static/img/identica.png +++ /dev/null diff --git a/source/static/img/logout.png b/source/static/img/logout.png Binary files differdeleted file mode 100644 index 55316f8b..00000000 --- a/source/static/img/logout.png +++ /dev/null diff --git a/source/static/img/management.png b/source/static/img/management.png Binary files differdeleted file mode 100644 index 7bcbc384..00000000 --- a/source/static/img/management.png +++ /dev/null diff --git a/source/static/img/mark-as-read.png b/source/static/img/mark-as-read.png Binary files differdeleted file mode 100644 index ffc90910..00000000 --- a/source/static/img/mark-as-read.png +++ /dev/null diff --git a/source/static/img/pinboard.png b/source/static/img/pinboard.png Binary files differdeleted file mode 100644 index 6dddc10b..00000000 --- a/source/static/img/pinboard.png +++ /dev/null diff --git a/source/static/img/previous-article.png b/source/static/img/previous-article.png Binary files differdeleted file mode 100644 index fcd9bfd8..00000000 --- a/source/static/img/previous-article.png +++ /dev/null diff --git a/source/static/img/reddit.png b/source/static/img/reddit.png Binary files differdeleted file mode 100755 index 2d615f2a..00000000 --- a/source/static/img/reddit.png +++ /dev/null diff --git a/source/static/img/scoopeo.png b/source/static/img/scoopeo.png Binary files differdeleted file mode 100755 index 052c7dc8..00000000 --- a/source/static/img/scoopeo.png +++ /dev/null diff --git a/source/static/img/tuxrss.png b/source/static/img/tuxrss.png Binary files differdeleted file mode 100644 index 6eef7595..00000000 --- a/source/static/img/tuxrss.png +++ /dev/null diff --git a/source/static/img/unread.png b/source/static/img/unread.png Binary files differdeleted file mode 100644 index d3a641c7..00000000 --- a/source/static/img/unread.png +++ /dev/null diff --git a/source/templates/article.html b/source/templates/article.html deleted file mode 100644 index c1fe41e5..00000000 --- a/source/templates/article.html +++ /dev/null @@ -1,67 +0,0 @@ -## article.html -<%inherit file="base.html"/> -<div> - <div style="width: 50%; overflow:hidden; text-align: justify; margin:0 auto"> - <h1><i><a href="${article['article_link']}">${article["article_title"]}</a></i> from <a href="/feed/${feed['feed_id']}">${feed["feed_title"]}</a></h1> - <br /> - %if article["article_like"]: - <a href="/like/0:${feed['feed_id']}:${article['article_id']}"><img src="/static/img/heart.png" title="I like this article!" /></a> - %else: - <a href="/like/1:${feed['feed_id']}:${article['article_id']}"><img src="/static/img/heart_open.png" title="Click if you like this article." /></a> - %endif - <a href="/delete_article/${feed['feed_id']}:${article['article_id']}"><img src="/static/img/cross.png" title="Delete this article" /></a> - <br /><br /> - ${description} - <div style="float:right;"> - <a href="/article/${feed['feed_id']}:${following['article_id']}" title="${following['article_title']}"> - <img src="/static/img/following-article.png" /> - </a> - </div> - <div style="float:left;"> - <a href="/article/${feed['feed_id']}:${previous['article_id']}" title="${previous['article_title']}"> - <img src="/static/img/previous-article.png" /> - </a> - </div> - <br /><br /><br /> - - %if plain_text == "1": - <a href="/article/${feed['feed_id']}:${article['article_id']}">HTML version</a> - %else: - <a href="/article/${feed['feed_id']}:${article['article_id']}/?plain_text=1">Plain text</a> - %endif - - <a href="/epub/${feed['feed_id']}:${article['article_id']}">Export to EPUB</a> - <br /> - - Share this article:<br /><br /> - <a href="javascript:(function(){f='https://${diaspora}/bookmarklet?url=${article['article_link']}&title=${article['article_title']}&notes=via pyAggr3g470r&v=1&';a=function(){if(!window.open(f+'noui=1&jump=doclose','diasporav1','location=yes,links=no,scrollbars=no,toolbar=no,width=620,height=250'))location.href=f+'jump=yes'};if(/Firefox/.test(navigator.userAgent)){setTimeout(a,0)}else{a()}})()"> - <img src="/static/img/diaspora.png" title="Share on Diaspora" /></a> - - <a href="http://identi.ca/index.php?action=newnotice&status_textarea=${article['article_title']}:${article['article_link']}" title="Share on Identi.ca" target="_blank"><img src="/static/img/identica.png" /></a> - - <a href="https://api.pinboard.in/v1/posts/add?url=${article['article_link']}&description=${article['article_title']}" - rel="noreferrer" target="_blank"> - <img src="/static/img/pinboard.png" title="Share on Pinboard" /></a> - - <a href="http://digg.com/submit?url=${article['article_link']}&title=${article['article_title']}" - rel="noreferrer" target="_blank"> - <img src="/static/img/digg.png" title="Share on Digg" /></a> - - <a href="http://reddit.com/submit?url=${article['article_link']}&title=${article['article_title']}" - rel="noreferrer" target="_blank"> - <img src="/static/img/reddit.png" title="Share on reddit" /></a> - - <a href="http://scoopeo.com/scoop/new?newurl=${article['article_link']}&title=${article['article_title']}" - rel="noreferrer" target="_blank"> - <img src="/static/img/scoopeo.png" title="Share on Scoopeo" /></a> - - <a href="http://blogmarks.net/my/new.php?url=${article['article_link']}&title=${article['article_title']}" - rel="noreferrer" target="_blank"> - <img src="/static/img/blogmarks.png" title="Share on Blogmarks" /></a> - - <g:plusone size="standard" count="true" href="${article['article_link']}"></g:plusone> - - <br /><br /> - <div align="center"> - <a href="/var/qrcode/${article['article_id']}.png"><img src="/var/qrcode/${article['article_id']}.png" title="Share with your smartphone" width="500" height="500" /></a> - </div> - </div> diff --git a/source/templates/articles.html b/source/templates/articles.html deleted file mode 100644 index d7fcc4d5..00000000 --- a/source/templates/articles.html +++ /dev/null @@ -1,43 +0,0 @@ -## articles.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="right inner"> - <a href="/mark_as_read/Feed:${feed['feed_id']}">Mark all articles from this feed as read</a> - <br /> - <form method=get action="/search/Feed${feed['feed_id']}"> - <input type="search" name="query" value="" placeholder="Search this feed" maxlength=2048 autocomplete="on"> - </form> - <hr /> -</div> - -<div class="left inner"> - <h1>Articles of the feed <i><a href="/feed/${feed['feed_id']}">${feed['feed_title']}</a></i></h1> - %if articles.count() == 0: - <p>No articles yet.</p> - %else: - <br /> - %endif - %for article in articles: - <% - if article["article_readed"] == False: - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - - if article["article_like"] == True: - like = """<img src="/static/img/heart.png" title="I like this article!" />""" - else: - like = "" - - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - %> - ${article["article_date"].strftime('%Y-%m-%d %H:%M')} - <a class="tooltip" href="/article/${feed['feed_id']}:${article['article_id']}" rel="noreferrer" target="_blank">${not_read_begin}${article["article_title"][:150]}${not_read_end}<span class="classic">${description}</span></a> ${like} - <br /> - %endfor - <h4><a href="/">All feeds</a></h4> diff --git a/source/templates/base.html b/source/templates/base.html deleted file mode 100644 index 03c40f90..00000000 --- a/source/templates/base.html +++ /dev/null @@ -1,28 +0,0 @@ -## base.html -<!DOCTYPE html> -<html> -<head> - <meta charset="utf-8" /> - %if header_text is UNDEFINED: - <title>pyAggr3g470r</title> - %elif header_text == 0: - <title>pyAggr3g470r</title> - %else: - <title>${header_text} - pyAggr3g470r</title> - %endif - <link rel="stylesheet" href="/static/css/style.css" /> - <script src="https://apis.google.com/js/plusone.js"></script> -</head> -<body> - <div class="right innerlogo"> - <a href="/"><img src="/static/img/tuxrss.png" title="What's new today?" /></a> - </div> - <a href="/"><h1 id="top">pyAggr3g470r</h1></a> - ${self.body()} - <hr /> - <p>This software is under GPLv3 license. You are welcome to copy, modify or - redistribute the source code according to the <a href="http://www.gnu.org/licenses/gpl-3.0.txt">GPLv3</a> license.<br /> - <a href="https://bitbucket.org/cedricbonhomme/pyaggr3g470r/" rel="noreferrer" target="_blank">Source code</a> of pyAggr3g470r.</p> - </div> -</body> -</html> diff --git a/source/templates/confirmation.html b/source/templates/confirmation.html deleted file mode 100644 index ae206838..00000000 --- a/source/templates/confirmation.html +++ /dev/null @@ -1,5 +0,0 @@ -## confirmation.html -<%inherit file="base.html"/> -<div class="left inner"> - <h1>Your request processed successfully:</h1> - <p>${message}</p> diff --git a/source/templates/error.html b/source/templates/error.html deleted file mode 100644 index 3790d3e9..00000000 --- a/source/templates/error.html +++ /dev/null @@ -1,5 +0,0 @@ -## error.html -<%inherit file="base.html"/> -<div class="left inner"> - <h1>An error occured:</h1> - ${message}
\ No newline at end of file diff --git a/source/templates/favorites.html b/source/templates/favorites.html deleted file mode 100644 index 4dd17b6b..00000000 --- a/source/templates/favorites.html +++ /dev/null @@ -1,30 +0,0 @@ -## favorites.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="left inner"> - <h1>Your favorites articles (${sum([elem.count() for elem in articles.values()])})</h1> - %for feed in feeds: - <% - new_feed_section = True - %> - %for article in articles[feed["feed_id"]]: - <% - if new_feed_section: - new_feed_section = False - title = """<h2><a name="%s"><a href="%s" rel="noreferrer"target="_blank">%s</a></a><a href="%s" rel="noreferrer" target="_blank"><img src="%s" width="28" height="28" /></a></h2>\n""" % \ - (feed["feed_id"], feed["site_link"], feed["feed_title"], feed["feed_link"], feed["feed_image"]) - else: - title = "" - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - %> - ${title} - - ${article["article_date"].strftime('%Y-%m-%d %H:%M')} - <a class="tooltip" href="/article/${feed['feed_id']}:${article['article_id']}" rel="noreferrer" target="_blank">${article["article_title"][:150]}<span class="classic">${description}</span></a><br /> - %endfor - %endfor diff --git a/source/templates/feed.html b/source/templates/feed.html deleted file mode 100644 index 2dadb8b4..00000000 --- a/source/templates/feed.html +++ /dev/null @@ -1,148 +0,0 @@ -## feed.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="left inner"> - %if articles != []: - <p>The feed <b>${feed['feed_title']}</b> contains <b>${format(nb_articles_feed, ',d')}</b> articles. - Representing ${round((nb_articles_feed / nb_articles_total) * 100, 4)} percent of the total (${format(nb_articles_total, ',d')} articles). - <br /> - Address of the feed: <a href="${feed['feed_link']}">${feed['feed_link']}</a>. - <br /> - Address of the site: <a href="${feed['site_link']}">${feed['site_link']}</a>. - <br /> - Logo: <img src="${feed['feed_image']}" width="28px" height="28px" /></p> - - <p>${(nb_unread_articles_feed == 0 and ["All articles are read"] or ['<a href="/unread/'+feed["feed_id"] + ' ">'+str(nb_unread_articles_feed)+'</a>' + ' unread article' + (nb_unread_articles_feed == 1 and [""] or ["s"])[0]])[0]}.</p> - %else: - <p>No articles for the feed <b>${feed['feed_title']}</b>. - <br /> - Address of the feed: <a href="${feed['feed_link']}">${feed['feed_link']}</a>. - <br /> - Address of the site: <a href="${feed['site_link']}">${feed['site_link']}</a>.</p> - %endif - - %if feed["mail"] == True: - <p> - You are receiving articles from this feed to the address: <a href="mail:${mail_to}">${mail_to}</a>. - <a href="/mail_notification/0:${feed['feed_id']}">Stop</a> receiving articles from this feed. - %if not mail_notification_enabled: - <br />However e-mail notification is disabled in the configuration file. - %endif - </p> - %endif - - %if articles != []: - <p>The last article was posted ${elapsed.days} day(s) ago.<br /> - Daily average: ${average}, between the ${first_post_date.strftime('%Y-%m-%d')} and the ${end_post_date.strftime('%Y-%m-%d')}.</p> - - <br /> - <h1>Recent articles</h1> - <% - html = "" - %> - %for article in articles: - <% - if article["article_readed"] == False: - # not readed articles are in bold - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - - # display a heart for faved articles - if article["article_like"] == True: - like = """ <img src="/static/img/heart.png" title="I like this article!" />""" - else: - like = "" - - # Descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - # Title of the article - article_title = article["article_title"] - if len(article_title) >= 80: - article_title = article_title[:80] + " ..." - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s%s%s<span class="classic">%s</span></a>""" % \ - (feed["feed_id"], article["article_id"], not_read_begin, \ - article_title, not_read_end, description) + like + "<br />\n" - %> - %endfor - ${html} - - <a href="/articles/${feed['feed_id']}">All articles</a> - <br /> - - %if nb_favorites != 0: - <br /></br /> - <h1>Your favorites articles for this feed</h1> - <% - html = "" - %> - %for article in favorites: - <% - #descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s<span class="classic">%s</span></a><br />\n""" % \ - (feed["feed_id"], article["article_id"], article["article_title"][:150], description) - %> - %endfor - ${html} - %endif - %endif - - - - <br /> - <h1>Edit this feed</h1> - <form method=post action="/change_feed_name/"> - <input type="text" name="new_feed_name" value="" placeholder="Enter a new name (then press Enter)." maxlength=2048 autocomplete="on" size="50" /> - <input type="hidden" name="feed_id" value="${feed['feed_id']}" /> - </form> - - <form method=post action="/change_site_url/"> - <input type="url" name="new_site_url" value="" placeholder="Enter a new URL for this site (then press Enter)." maxlength=2048 autocomplete="on" size="50" /> - <input type="hidden" name="feed_id" value="${feed['feed_id']}" /> - <input type="hidden" name="old_site_url" value="${feed['site_link']}" /> - </form> - - <form method=post action="/change_feed_url/"> - <input type="url" name="new_feed_url" value="" placeholder="Enter a new URL in order to retrieve articles (then press Enter)." maxlength=2048 autocomplete="on" size="50" /> - <input type="hidden" name="feed_id" value="${feed['feed_id']}" /> - <input type="hidden" name="old_feed_url" value="${feed['feed_link']}" /> - </form> - - <form method=post action="/change_feed_logo/"> - <input type="text" name="new_feed_logo" value="" placeholder="Enter the URL of the logo (then press Enter)." maxlength=2048 autocomplete="on" size="50" /> - <input type="hidden" name="feed_id" value="${feed['feed_id']}" /> - </form> - - <form method=get action="/remove_feed/${feed['feed_id']}"> - <p><input type="submit" value="Unsubscribe" /> - (deletes corresponding articles)</p> - </form> - - %if articles != []: - </br /> - <h1>Tag cloud</h1> - <form method=get action="/feed/${feed['feed_id']}"> - Minimum size of a word: - <input type="number" name="word_size" value="${word_size}" min="2" max="15" step="1" size="2"> - </form> - <div style="width: 35%; overflow:hidden; text-align: justify">${tag_cloud}</div> - %endif diff --git a/source/templates/history.html b/source/templates/history.html deleted file mode 100644 index 16f909dc..00000000 --- a/source/templates/history.html +++ /dev/null @@ -1,80 +0,0 @@ -## history.html -<%inherit file="base.html"/> -<% -import utils -import calendar -from collections import Counter -%> -<div class="left inner"> - <% - html = "" - # Get the date from the tag cloud - # Format: /history/?query=year:2011-month:06 to get the - # list of articles of June, 2011. - if query == "all": - html += "<h1>Search with tags cloud</h1>\n" - html += "<h4>Choose a year</h4>\n" - if "year" in query: - the_year = query.split('-')[0].split(':')[1] - if "month" not in query: - html += "<h1>Choose a month for the year " + the_year + "</h1>\n" - if "month" in query: - the_month = query.split('-')[1].split(':')[1] - html += "<h1>Articles of "+ calendar.month_name[int(the_month)] + ", "+ the_year +".</h1>\n" - - timeline = Counter() - for feed in feeds: - new_feed_section = True - for article in mongo.get_articles(feed["feed_id"]): - - if query == "all": - timeline[article["article_date"].strftime('%Y')] += 1 - - elif query[:4] == "year": - - if article["article_date"].strftime('%Y') == the_year: - timeline[article["article_date"].strftime('%m')] += 1 - - if "month" in query: - if article["article_date"].strftime('%m') == the_month: - if article["article_readed"] == False: - # not readed articles are in bold - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - - if article["article_like"] == True: - like = """ <img src="/static/img/heart.png" title="I like this article!" />""" - else: - like = "" - # Descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - # Title of the article - article_title = article["article_title"] - if len(article_title) >= 80: - article_title = article_title[:80] + " ..." - - if new_feed_section is True: - new_feed_section = False - html += """<h2><a name="%s"><a href="%s" rel="noreferrer" - target="_blank">%s</a></a><a href="%s" rel="noreferrer" - target="_blank"><img src="%s" width="28" height="28" /></a></h2>\n""" % \ - (feed["feed_id"], feed["site_link"], feed["feed_title"], feed["feed_link"], feed["feed_image"]) - - html += article["article_date"].strftime("%a %d (%H:%M:%S) ") + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s%s%s<span class="classic">%s</span></a>""" % \ - (feed["feed_id"], article["article_id"], not_read_begin, \ - article_title, not_read_end, description) + like + "<br />\n" - if query == "all": - query_string = "year" - elif "year" in query: - query_string = "year:" + the_year + "-month" - if "month" not in query: - html += '<div style="width: 35%; overflow:hidden; text-align: justify">' + \ - utils.tag_cloud([(elem, timeline[elem]) for elem in timeline.keys()], query_string) + '</div>' - %> - ${html} diff --git a/source/templates/inactives.html b/source/templates/inactives.html deleted file mode 100644 index 57482b61..00000000 --- a/source/templates/inactives.html +++ /dev/null @@ -1,15 +0,0 @@ -## inactives.html -<%inherit file="base.html"/> -<div class="left inner"> - %if inactives != []: - <form method=get action="/inactives/"> - <h1>Feeds with no recent articles since <input type="number" name="nb_days" value="${nb_days}" min="0" max="1000000" step="1" size="4" style="text-align: center" /> days:</h1> - </form> - <ul> - %for item in inactives: - <li><a href="/feed/${item[0]["feed_id"]}">${item[0]["feed_title"]}</a> (${item[1].days} days)</li> - %endfor - </ul> - %else: - <p>No inactive feeds.<p> - %endif diff --git a/source/templates/index.html b/source/templates/index.html deleted file mode 100644 index 88ca7a87..00000000 --- a/source/templates/index.html +++ /dev/null @@ -1,108 +0,0 @@ -## index.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="right inner"> - <form method=get action="/search/"> - <input type="search" name="query" value="" placeholder="Search articles" maxlength=2048 autocomplete="on" /> - </form> - <div class="nav_container"><div align="center"><i><a href="/subscriptions/">Subscriptions</a></i> (${nb_feeds})<br /></div> - <% - html = "" - %> - %for feed in feeds: - <% - if mongo.nb_unread_articles(feed["feed_id"]) != 0: - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - html += """<div style='float:left'><a href="/#%s">%s</a></div> - <div style='float:right'> (<a href="/unread/%s" title="Unread article(s)">%s%s%s</a> / %s)</div> - <div style="clear:both"></div>\n""" % \ - (feed["feed_id"], feed["feed_title"], feed["feed_id"], not_read_begin, \ - format(mongo.nb_unread_articles(feed["feed_id"]), ',d'), not_read_end, format(mongo.nb_articles(feed["feed_id"]), ',d')) - %> - %endfor - ${html} - </div> -</div> - -<div class="left inner"> - <div class="menu_container"> - %if feeds: - <a href="/management/"><img src="/static/img/management.png" title="Management" /></a> - <a href="/history/"><img src="/static/img/history.png" title="History" /></a> - - <a href="/favorites/"><img src="/static/img/heart-32x32.png" title="Your favorites (${nb_favorites})" /></a> - <a href="/notifications/"><img src="/static/img/email-follow.png" title="Active e-mail notifications (${nb_mail_notifications})" /></a> - - %if nb_unread_articles != 0: - <a href="/mark_as_read/"><img src="/static/img/mark-as-read.png" title="Mark articles as read" /></a> - <a href="/unread/"><img src="/static/img/unread.png" title="Unread article(s): ${nb_unread_articles}" /></a> - %endif - %endif - <a href="/fetch/"><img src="/static/img/check-news.png" title="Check for news" /></a> - - <a href="/logout/"><img src="/static/img/logout.png" title="Logout" /></a> - </div><br/> - <% - html = "" - %> - <% - for feed in feeds: - html += """\n<h2 id="%s"><a href="%s" rel="noreferrer" target="_blank">%s</a> - <a href="%s" rel="noreferrer" target="_blank"> - <img src="%s" width="28" height="28" /> - </a> - </h2>\n<br />""" % \ - (feed["feed_id"], feed["site_link"], feed["feed_title"], \ - feed["feed_link"], feed["feed_image"]) - - # The main page display only 10 articles by feeds. - for article in mongo.get_articles(feed["feed_id"], limit=10): - if article["article_readed"] == False: - # not readed articles are in bold - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - - # display a heart for faved articles - if article["article_like"] == True: - like = """ <img src="/static/img/heart.png" title="I like this article!" />""" - else: - like = "" - - # Descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content.split(' ')[:55]) - else: - description = "No description." - # Title of the article - article_title = article["article_title"] - if len(article_title) >= 80: - article_title = article_title[:80] + " ..." - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s%s%s<span class="classic">%s</span></a>""" % \ - (feed["feed_id"], article["article_id"], not_read_begin, \ - article_title, not_read_end, description) + like + "<br />\n" - html += "<br />\n" - - # some options for the current feed - html += """<a href="/articles/%s">All articles</a> """ % (feed["feed_id"],) - html += """<a href="/feed/%s">Feed summary</a> """ % (feed["feed_id"],) - html += """<div class="right"><h2><a href="/fetch/%s"><img src="/static/img/check-news.png" title="Check this feed for news" /></a></h2></div>\n""" % (feed["feed_id"],) - if mongo.nb_unread_articles(feed["feed_id"]) != 0: - html += """ <a href="/mark_as_read/">Mark all as read</a>""" - html += """ <a href="/unread/%s">%s unread article(s)</a>""" % (feed["feed_id"], mongo.nb_unread_articles(feed["feed_id"])) - if feed["mail"] == False: - html += """<br />\n<a href="/mail_notification/1:%s" title="By e-mail">Enable email notifications</a>""" % (feed["feed_id"],) - else: - html += """<br />\n<a href="/mail_notification/0:%s" title="By e-mail">Disable email notifications</a>""" % (feed["feed_id"],) - html += """<h4><a href="/#top">Top</a></h4>\n""" - %> - ${html} diff --git a/source/templates/languages.html b/source/templates/languages.html deleted file mode 100644 index d186677d..00000000 --- a/source/templates/languages.html +++ /dev/null @@ -1,25 +0,0 @@ -## languages.html -<%inherit file="base.html"/> -<div class="left inner"> - <h1>Summary</h1> - <ul> - %for language in articles_sorted_by_languages.keys(): - <li><a href="#${language}">${language}</a>: ${sum(map(len, articles_sorted_by_languages[language].values()))} articles</li> - %endfor - </ul> - %for language in articles_sorted_by_languages.keys(): - <br /> - <h1 id="${language}">${language}</h1> - %for feed_id in articles_sorted_by_languages[language]: - <% - feed = mongo.get_feed(feed_id) - %> - <h2>${feed["feed_title"]}</h2> - %for article in articles_sorted_by_languages[language][feed_id][:10]: - ${article["article_date"].strftime('%Y-%m-%d %H:%M')} - <a href="/article/${feed['feed_id']}:${article['article_id']}" rel="noreferrer" target="_blank">${article["article_title"]}</a> - <br /> - %endfor - <br /> - %endfor - <br /> - %endfor
\ No newline at end of file diff --git a/source/templates/management.html b/source/templates/management.html deleted file mode 100644 index f6c68ace..00000000 --- a/source/templates/management.html +++ /dev/null @@ -1,84 +0,0 @@ -## management.html -<%inherit file="base.html"/> -<div class="left inner"> - <h1>Subscriptions</h1> - <p>Add a new subscription (current <a href="/subscriptions/">subscriptions</a>):</p> - <form method=get action="/add_feed/"> - <input type="url" name="url" placeholder="URL of a site or feed." maxlength=2048 autocomplete="off" /> - <input type="submit" value="OK" /> - </form> - - %if feeds: - <p>Unsubscribe from a source (deletes corresponding articles):</p> - <form method=get action="/remove_feed/"> - <select name="feed_id"> - %for feed in feeds: - <option value="${feed['feed_id']}">${feed['feed_title']}</option> - %endfor - </select> - <input type="submit" value="OK" /> - </form> - %endif - - %if not mail_notification_enabled: - <p>E-mail notification is disabled in the configuration file.</p> - %endif - - %if feeds: - <hr /> - <h1>Facts</h1> - <ul> - <li>active e-mail notifications: <a href="/notifications/">${nb_mail_notifications}</a>;</li> - <li>you like <a href="/favorites/">${nb_favorites}</a> article(s);</li> - <li><a href="/statistics/">tag clouds</a>;</li> - <li><a href="/inactives/">inactive feeds</a>;</li> - <li><a href="/languages/">languages</a>.</li> - </ul> - %endif - - <hr /> - - <h1>Account</h1> - <p> - <form method=get action="/change_username/"> - <input type="text" name="new_username" value="" placeholder="Enter a new username." /> - </form> - <br /> - <form method=get action="/change_password/"> - <input type="password" name="new_password" value="" placeholder="Enter a new password." /> - </form> - </p> - - <hr /> - - - <h1>Database</h1> - <p>${nb_articles} article(s) are stored in the database with <a href="/unread/">${nb_unread_articles} unread article(s)</a>. - - <form method=get action="/fetch/"> - <input type="submit" value="Fetch all feeds" /> - </form> - <form method=get action="/drop_base"> - <input type="submit" value="Delete all articles" /> - </form> - - <br /> - <form method=get action="/index_base"> - <input type="submit" value="Index database" /> (${nb_indexed_documents} indexed documents) - </form> - - - <hr /> - - <h1>Export articles</h1> - <p> - <form method=get action="/export/"> - <select name="export_method"> - <option value="export_html" selected='selected'>HTML (simple Webzine)</option> - <option value="export_epub">ePub</option> - <option value="export_pdf">PDF</option> - <option value="export_txt">Text</option> - </select> - <input type="submit" value="Export" /> - </form> - </p> diff --git a/source/templates/notifications.html b/source/templates/notifications.html deleted file mode 100644 index 88558b05..00000000 --- a/source/templates/notifications.html +++ /dev/null @@ -1,18 +0,0 @@ -## article.html -<%inherit file="base.html"/> -<div class="left inner"> - %if feeds != []: - <h1>You are receiving e-mails for the following feeds:</h1> - <ul> - %for feed in feeds: - <li><a href="/feed/${feed['feed_id']}">${feed['feed_title']}</a> - <a href="/mail_notification/0:${feed['feed_id']}">Stop</a></li> - %endfor - </ul> - %else: - <p>No active notifications.<p> - %endif - <p>Notifications are sent to: <a href="mail:${mail_to}">${mail_to}</a>. - %if not mail_notification_enabled: - However e-mail notification is disabled in the configuration file. - %endif - </p> diff --git a/source/templates/search.html b/source/templates/search.html deleted file mode 100644 index 435ef443..00000000 --- a/source/templates/search.html +++ /dev/null @@ -1,57 +0,0 @@ -## search.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="left inner"> -%if len(search_result) != 0: - <h1>Articles containing the string <i>${query}</i> (${sum([len(articles) for articles in search_result.values()])} results)</h1> -%else: - <h1>String <i>${query}</i> not found.</h1> -%endif -<br /> -<form method=get action="/index_base"> - <input type="submit" value="Reindex database" /> -</form> - -<% - html = "" - feed_id = None -%> - -%for feed_id in search_result.keys(): - <% - new_feed_section = True - feed = mongo.get_feed(feed_id) - for article in search_result[feed["feed_id"]]: - if new_feed_section is True: - new_feed_section = False - html += """<h2><a href="/articles/%s" rel="noreferrer" target="_blank">%s</a><a href="%s" rel="noreferrer" target="_blank"><img src="%s" width="28" height="28" /></a></h2>\n""" % \ - (feed["feed_id"], feed["feed_title"], feed["feed_link"], feed["feed_image"]) - - if article["article_readed"] == False: - # not readed articles are in bold - not_read_begin, not_read_end = "<b>", "</b>" - else: - not_read_begin, not_read_end = "", "" - - # display a heart for faved articles - if article["article_like"] == True: - like = """ <img src="/img/heart.png" title="I like this article!" />""" - else: - like = "" - - # descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - description = " ".join(article_content[:500].split(' ')[:-1]) - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s%s%s<span class="classic">%s</span></a>""" % \ - (feed["feed_id"], article["article_id"], not_read_begin, \ - article["article_title"][:150], not_read_end, description) + like + "<br />\n" - %> -%endfor - -${html}
\ No newline at end of file diff --git a/source/templates/statistics.html b/source/templates/statistics.html deleted file mode 100644 index 5dfcbfa8..00000000 --- a/source/templates/statistics.html +++ /dev/null @@ -1,14 +0,0 @@ -## statistics.html -<%inherit file="base.html"/> -<div class="left inner"> - %if articles: - <h1>Statistics</h1> - <h3>Tag cloud</h3> - <form method=get action="/statistics/"> - Minimum size of a word: - <input type="number" name="word_size" value="${word_size}" min="2" max="15" step="1" size="2" /> - </form> - <div style="width: 35%; overflow:hidden; text-align: justify"> - ${tag_cloud} - </div> - %endif
\ No newline at end of file diff --git a/source/templates/subscriptions.html b/source/templates/subscriptions.html deleted file mode 100644 index 8cee08ab..00000000 --- a/source/templates/subscriptions.html +++ /dev/null @@ -1,13 +0,0 @@ -## subscriptions.html -<%inherit file="base.html"/> -<div class="left inner"> - <h1>Subscriptions</h1> - %if feeds: - <ul> - %for feed in feeds: - <li><a href="/feed/${feed['feed_id']}">${feed['feed_title']}</a></li> - %endfor - </ul> - %else: - <p>No subscriptions.</p> - %endif
\ No newline at end of file diff --git a/source/templates/unread.html b/source/templates/unread.html deleted file mode 100644 index 93df53e7..00000000 --- a/source/templates/unread.html +++ /dev/null @@ -1,76 +0,0 @@ -## unread.html -<%inherit file="base.html"/> -<% -import utils -%> -<div class="left inner"> - <% - html = "" - if mongo.nb_unread_articles() != 0: - - # List unread articles of all the database - if feed_id == "": - html += "<h1>Unread article(s)</h1>" - html += """\n<br />\n<a href="/mark_as_read/">Mark all articles as read</a>\n<hr />\n""" - for feed in feeds: - new_feed_section = True - nb_unread = 0 - - # For all unread article of the current feed. - for article in mongo.get_articles(feed["feed_id"], condition=("article_readed", False)): - nb_unread += 1 - if new_feed_section is True: - new_feed_section = False - html += """<h2><a name="%s"><a href="%s" rel="noreferrer" target="_blank">%s</a></a><a href="%s" rel="noreferrer" target="_blank"><img src="%s" width="28" height="28" /></a></h2>\n""" % \ - (feed["feed_id"], feed["site_link"], feed["feed_title"], feed["feed_link"], feed["feed_image"]) - - # descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s<span class="classic">%s</span></a><br />\n""" % \ - (feed["feed_id"], article["article_id"], article["article_title"][:150], description) - - if nb_unread == mongo.nb_unread_articles(feed["feed_id"]): - html += """<br />\n<a href="/mark_as_read/Feed:%s">Mark all as read</a>\n""" % \ - (feed["feed_id"],) - html += """<hr />\n<a href="/mark_as_read/">Mark articles as read</a>\n""" - - - # List unread articles of a feed - else: - try: - feed = mongo.get_feed(feed_id) - except: - return "<p>This feed do not exists.</p>" - html += """<h1>Unread article(s) of the feed <a href="/articles/%s">%s</a></h1> - <br />""" % (feed_id, feed["feed_title"]) - - # For all unread article of the feed. - for article in mongo.get_articles(feed_id, condition=("article_readed", False)): - # descrition for the CSS ToolTips - article_content = utils.clear_string(article["article_content"]) - if article_content: - description = " ".join(article_content[:500].split(' ')[:-1]) - else: - description = "No description." - - # a description line per article (date, title of the article and - # CSS description tooltips on mouse over) - html += article["article_date"].strftime('%Y-%m-%d %H:%M') + " - " + \ - """<a class="tooltip" href="/article/%s:%s" rel="noreferrer" target="_blank">%s<span class="classic">%s</span></a><br />\n""" % \ - (feed_id, article["article_id"], article["article_title"][:150], description) - - html += """<hr />\n<a href="/mark_as_read/Feed:%s">Mark all as read</a>""" % (feed_id,) - # No unread article - else: - html += '<h1>No unread article(s)</h1>\n<br />\n<a href="/fetch/">Why not check for news?</a>' - html += """\n<h4><a href="/">All feeds</a></h4>""" - %> - ${html} diff --git a/source/testbinarytree.py b/source/testbinarytree.py deleted file mode 100644 index 84670ca1..00000000 --- a/source/testbinarytree.py +++ /dev/null @@ -1,45 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -import time -import sys -import resource -# Increases Python's recursion limit and the size of the stack. -resource.setrlimit(resource.RLIMIT_STACK, (2**29,-1)) -sys.setrecursionlimit(10**6) - -import mongodb -import binarytree -import conf - -print("Loading articles from the database...") -database = mongodb.Articles(conf.MONGODB_ADDRESS, conf.MONGODB_PORT, \ - conf.MONGODB_DBNAME, conf.MONGODB_USER, \ - conf.MONGODB_PASSWORD) -begin = time.time() -articles = database.get_articles() -end = time.time() -print(("{} articles loaded in {} seconds.".format(len(articles), end-begin))) - -print("Generating the binary tree...") -begin = time.time() -root = binarytree.Node(articles[0]) -tree = binarytree.OrderedBinaryTree(root) -# add the root node (first article of the list) -#root = tree.addNode(articles[0]) -for article in articles[1:]: - tree.insert(tree.root, article) -end = time.time() -print(("Generation done in {0:2f} seconds.".format(end-begin))) - -print("Maximum depth of the tree:") -print(tree.maxDepth(tree.root)) -print("Oldest article:") -oldest_article = tree.minValue(tree.root) -print((oldest_article["article_date"].strftime('%Y-%m-%d %H:%M') + \ - " - " + oldest_article["article_title"])) -print("Newest article:") -newest_article = tree.maxValue(tree.root) -print((newest_article["article_date"].strftime('%Y-%m-%d %H:%M') + \ - " - " + newest_article["article_title"])) -#print(tree)
\ No newline at end of file diff --git a/source/utils.py b/source/utils.py deleted file mode 100755 index d39e402f..00000000 --- a/source/utils.py +++ /dev/null @@ -1,317 +0,0 @@ -#! /usr/bin/env python -#-*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/> - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 1.5 $" -__date__ = "$Date: 2010/12/07 $" -__revision__ = "$Date: 2013/07/24 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -# -# This file provides functions used for: -# - the database management; -# - generation of tags cloud; -# - HTML processing; -# - e-mail notifications. -# - -import os -import re -import glob -import operator -import calendar -import html.entities - -try: - from qrcode.pyqrnative.PyQRNative import QRCode, QRErrorCorrectLevel, CodeOverflowException - from qrcode import qr -except: - pass - -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -import urllib.request, urllib.error, urllib.parse -import http.server -from bs4 import BeautifulSoup - -from collections import Counter -from contextlib import contextmanager - -import conf - -# regular expression to check URL -url_finders = [ \ - re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \ - re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?"), \ - re.compile("(~/|/|\\./)([-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]|\\\\)+"), \ - re.compile("'\\<((mailto:)|)[-A-Za-z0-9\\.]+@[-A-Za-z0-9\\.]+") \ -] - -import log -pyaggr3g470r_log = log.Log() - -@contextmanager -def opened_w_error(filename, mode="r"): - try: - f = open(filename, mode) - except IOError as err: - yield None, err - else: - try: - yield f, None - finally: - f.close() - -def open_url(url): - """ - Open an URL with the proxy and the user-agent - specified in the configuration file. - """ - if conf.HTTP_PROXY == "": - proxy = {} - else: - proxy = {"http" : conf.HTTP_PROXY} - opener = urllib.request.FancyURLopener(proxy) - try: - opener = urllib.request.build_opener() - opener.addheaders = [('User-agent', conf.USER_AGENT)] - return (True, opener.open(url)) - except urllib.error.HTTPError as e: - # server couldn't fulfill the request - error = (url, e.code, \ - http.server.BaseHTTPRequestHandler.responses[e.code][1]) - pyaggr3g470r_log.error(url + " " + str(e.code) + " " + \ - http.server.BaseHTTPRequestHandler.responses[e.code][1]) - return (False, error) - except urllib.error.URLError as e: - # failed to reach the server - if type(e.reason) == str: - error = (url, e.reason, e.reason) - pyaggr3g470r_log.error(url + " " + e.reason) - else: - error = (url, e.reason.errno, e.reason.strerror) - pyaggr3g470r_log.error(url + " " + str(e.reason.errno) + " " + \ - e.reason.strerror) - return (False, error) - -def generate_qr_code(article): - """ - Generated a QR Code for the article given in parameter. - """ - try: - os.makedirs("./var/qrcode/") - except OSError: - pass - if not os.path.isfile("./var/qrcode/" + article["article_id"] + ".png"): - # QR Code generation - try: - f = qr.QRUrl(url = article["article_link"]) - f.make() - f.save("./var/qrcode/" + article["article_id"] + ".png") - except: - pass - -def clear_string(data): - """ - Clear a string by removing HTML tags, HTML special caracters - and consecutive white spaces (more that one). - """ - p = re.compile(b'<[^>]+>') # HTML tags - q = re.compile(b'\s') # consecutive white spaces - return p.sub(b'', q.sub(b' ', bytes(data, "utf-8"))).decode("utf-8", "strict") - -def normalize_filename(name): - """ - Normalize a file name. - """ - file_name = re.sub("[,'!?|&]", "", name) - file_name = re.sub("[\s.]", "_", file_name) - file_name = file_name.strip('_') - file_name = file_name.strip('.') - return os.path.normpath(file_name) - -def load_stop_words(): - """ - Load the stop words and return them in a list. - """ - stop_words_lists = glob.glob('./var/stop_words/*.txt') - stop_words = [] - - for stop_wods_list in stop_words_lists: - with opened_w_error(stop_wods_list, "r") as (stop_wods_file, err): - if err: - stop_words = [] - else: - stop_words += stop_wods_file.read().split(";") - return stop_words - -def top_words(articles, n=10, size=5): - """ - Return the n most frequent words in a list. - """ - stop_words = load_stop_words() - words = Counter() - wordre = re.compile(r'\b\w{%s,}\b' % size, re.I) - for article in articles: - for word in [elem.lower() for elem in - wordre.findall(clear_string(article["article_content"])) \ - if elem.lower() not in stop_words]: - words[word] += 1 - return words.most_common(n) - -def tag_cloud(tags, query="word_count"): - """ - Generates a tags cloud. - """ - tags.sort(key=operator.itemgetter(0)) - if query == "word_count": - # tags cloud from the management page - return ' '.join([('<font size=%d><a href="/search/?query=%s" title="Count: %s">%s</a></font>\n' % \ - (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), word, format(count, ',d'), word)) \ - for (word, count) in tags]) - if query == "year": - # tags cloud for the history - return ' '.join([('<font size=%d><a href="/history/?query=%s:%s" title="Count: %s">%s</a></font>\n' % \ - (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), query, word, format(count, ',d'), word)) \ - for (word, count) in tags]) - return ' '.join([('<font size=%d><a href="/history/?query=%s:%s" title="Count: %s">%s</a></font>\n' % \ - (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), query, word, format(count, ',d'), calendar.month_name[int(word)])) \ - for (word, count) in tags]) - -def send_mail(mfrom, mto, feed_title, article_title, description): - """ - Send the article via mail. - """ - # Create the body of the message (a plain-text and an HTML version). - html = """<html>\n<head>\n<title>%s</title>\n</head>\n<body>\n%s\n</body>\n</html>""" % \ - (feed_title + ": " + article_title, description) - text = clear_string(description) - - # Create message container - the correct MIME type is multipart/alternative. - msg = MIMEMultipart('alternative') - msg['Subject'] = '[pyAggr3g470r] ' + feed_title + ": " + article_title - msg['From'] = mfrom - msg['To'] = mto - - # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(text, 'plain', 'utf-8') - part2 = MIMEText(html, 'html', 'utf-8') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg.attach(part1) - msg.attach(part2) - - # Send the message via local SMTP server. - try: - s = smtplib.SMTP(conf.smtp_server) - s.login(conf.username, conf.password) - except Exception as e: - print(e) - else: - s.send_message(msg) - s.quit() - -def add_feed(feed_url): - """ - Add the URL feed_url in the file feed.lst. - """ - with opened_w_error(conf.FEED_LIST, "r") as (f, err): - if err: - return False - else: - lines = f.readlines() - lines = list(map(str.strip, lines)) - if feed_url in lines: - return False - lines.append(feed_url) - with open(conf.FEED_LIST, "w") as f: - f.write("\n".join(lines)) - return True - -def change_feed_url(old_feed_url, new_feed_url): - """ - Change the URL of a feed given in parameter. - """ - # Replace the URL in the text file - with opened_w_error(conf.FEED_LIST, "r") as (f, err): - if err: - return False - else: - lines = f.readlines() - lines = list(map(str.strip, lines)) - try: - lines[lines.index(old_feed_url)] = new_feed_url - except: - return False - with opened_w_error(conf.FEED_LIST, "w") as (f, err): - if err: - return False - else: - f.write("\n".join(lines)) - return True - -def remove_feed(feed_url): - """ - Remove a feed from the file feed.lst and from the database. - """ - with opened_w_error(conf.FEED_LIST, "r") as (f, err): - if err: - return False - else: - lines = f.readlines() - lines = list(map(str.strip, lines)) - try: - del lines[lines.index(feed_url)] - except: - return False - with opened_w_error(conf.FEED_LIST, "w") as (f, err): - if err: - return False - else: - f.write("\n".join(lines)) - return True - -def search_feed(url): - """ - Search a feed in a HTML page. - """ - soup, page = None, None - try: - result = open_url(url) - if result[0] == True: - page = open_url(url)[1] - else: - return None - soup = BeautifulSoup(page) - except: - return None - feed_links = soup('link', type='application/atom+xml') - feed_links.extend(soup('link', type='application/rss+xml')) - for feed_link in feed_links: - if url not in feed_link['href']: - return urllib.parse.urljoin(url, feed_link['href']) - return feed_link['href'] - return None diff --git a/source/var/english-stop-words.txt b/source/var/english-stop-words.txt deleted file mode 100644 index 497a1f96..00000000 --- a/source/var/english-stop-words.txt +++ /dev/null @@ -1,311 +0,0 @@ - - | An English stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | Many of the forms below are quite rare (e.g. "yourselves") but included for - | completeness. - - | PRONOUNS FORMS - | 1st person sing - -i | subject, always in upper case of course - -me | object -my | possessive adjective - | the possessive pronoun `mine' is best suppressed, because of the - | sense of coal-mine etc. -myself | reflexive - | 1st person plural -we | subject - -| us | object - | care is required here because US = United States. It is usually - | safe to remove it if it is in lower case. -our | possessive adjective -ours | possessive pronoun -ourselves | reflexive - | second person (archaic `thou' forms not included) -you | subject and object -your | possessive adjective -yours | possessive pronoun -yourself | reflexive (singular) -yourselves | reflexive (plural) - | third person singular -he | subject -him | object -his | possessive adjective and pronoun -himself | reflexive - -she | subject -her | object and possessive adjective -hers | possessive pronoun -herself | reflexive - -it | subject and object -its | possessive adjective -itself | reflexive - | third person plural -they | subject -them | object -their | possessive adjective -theirs | possessive pronoun -themselves | reflexive - | other forms (demonstratives, interrogatives) -what -which -who -whom -this -that -these -those - - | VERB FORMS (using F.R. Palmer's nomenclature) - | BE -am | 1st person, present -is | -s form (3rd person, present) -are | present -was | 1st person, past -were | past -be | infinitive -been | past participle -being | -ing form - | HAVE -have | simple -has | -s form -had | past -having | -ing form - | DO -do | simple -does | -s form -did | past -doing | -ing form - - | The forms below are, I believe, best omitted, because of the significant - | homonym forms: - - | He made a WILL - | old tin CAN - | merry month of MAY - | a smell of MUST - | fight the good fight with all thy MIGHT - - | would, could, should, ought might however be included - - | | AUXILIARIES - | | WILL - |will - -would - - | | SHALL - |shall - -should - - | | CAN - |can - -could - - | | MAY - |may - |might - | | MUST - |must - | | OUGHT - -ought - - | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing - | pronoun + verb - -i'm -you're -he's -she's -it's -we're -they're -i've -you've -we've -they've -i'd -you'd -he'd -she'd -we'd -they'd -i'll -you'll -he'll -she'll -we'll -they'll - - | verb + negation - -isn't -aren't -wasn't -weren't -hasn't -haven't -hadn't -doesn't -don't -didn't - - | auxiliary + negation - -won't -wouldn't -shan't -shouldn't -can't -cannot -couldn't -mustn't - - | miscellaneous forms - -let's -that's -who's -what's -here's -there's -when's -where's -why's -how's - - | rarer forms - - | daren't needn't - - | doubtful forms - - | oughtn't mightn't - - | ARTICLES -a -an -the - - | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so - | high, that classification is pointless.) -and -but -if -or -because -as -until -while - -of -at -by -for -with -about -against -between -into -through -during -before -after -above -below -to -from -up -down -in -out -on -off -over -under - -again -further -then -once - -here -there -when -where -why -how - -all -any -both -each -few -more -most -other -some -such - -no -nor -not -only -own -same -so -than -too -very - - | Just for the record, the following words are among the commonest in English - - | one - | every - | least - | less - | many - | now - | ever - | never - | say - | says - | said - | also - | get - | go - | goes - | just - | made - | make - | put - | see - | seen - | whether - | like - | well - | back - | even - | still - | way - | take - | since - | another - | however - | two - | three - | four - | five - | first - | second - | new - | old - | high - | long
\ No newline at end of file diff --git a/source/var/feed.lst b/source/var/feed.lst deleted file mode 100755 index d9e6c924..00000000 --- a/source/var/feed.lst +++ /dev/null @@ -1,35 +0,0 @@ -http://www.foo.be/cgi-bin/wiki.pl?action=journal&tile=AdulauMessyDesk -http://blog.cedricbonhomme.org/feed/ -http://standblog.org/blog/feed/atom -http://www.haypocalc.com/blog/rss.php -http://linuxfr.org/news.atom -http://rss.slashdot.org/Slashdot/slashdot -http://theinvisiblethings.blogspot.com/feeds/posts/default -http://torvalds-family.blogspot.com/feeds/posts/default -http://www.python.org/channews.rdf -http://www.kde.org/dotkdeorg.rdf -http://feeds.feedburner.com/internetactu/bcmJ -http://www.april.org/fr/rss.xml -http://www.framablog.org/index.php/feed/atom -http://tarekziade.wordpress.com/feed/ -http://lwn.net/headlines/newrss -http://kernelnewbies.org/RecentChanges?action=rss_rc&ddiffs=1&unique=1 -http://www.kroah.com/log/index.rss -http://www.jeffersonswheel.org/?feed=rss2 -http://www.laquadrature.net/en/rss.xml -http://static.fsf.org/fsforg/rss/blogs.xml -http://esr.ibiblio.org/?feed=rss2 -http://www.schneier.com/blog/index.rdf -http://python-history.blogspot.com/feeds/posts/default -http://www.haypocalc.com/wordpress/feed -http://www.crypto.com/blog/rss10.xml -http://spaf.wordpress.com/feed/ -http://neopythonic.blogspot.com/feeds/posts/default -http://www.quuxlabs.com/feed/ -http://etbe.coker.com.au/feed/ -http://0pointer.de/blog/index.rss2 -http://www.bortzmeyer.org/feed-full.atom -http://blog.cr0.org/feeds/posts/default -http://sysc.tl/feed/ -http://planetKDE.org/rss20.xml -http://www.brankovukelic.com/feeds/posts/default
\ No newline at end of file diff --git a/source/var/french-stop-words.txt b/source/var/french-stop-words.txt deleted file mode 100644 index 08a2f5d7..00000000 --- a/source/var/french-stop-words.txt +++ /dev/null @@ -1,176 +0,0 @@ - - | A French stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -au | a + le -aux | a + les -avec | with -ce | this -ces | these -dans | with -de | of -des | de + les -du | de + le -elle | she -en | `of them' etc -et | and -eux | them -il | he -je | I -la | the -le | the -leur | their -lui | him -ma | my (fem) -mais | but -me | me -même | same; as in moi-même (myself) etc -mes | me (pl) -moi | me -mon | my (masc) -ne | not -nos | our (pl) -notre | our -nous | we -on | one -ou | where -par | by -pas | not -pour | for -qu | que before vowel -que | that -qui | who -sa | his, her (fem) -se | oneself -ses | his (pl) -son | his, her (masc) -sur | on -ta | thy (fem) -te | thee -tes | thy (pl) -toi | thee -ton | thy (masc) -tu | thou -un | a -une | a -vos | your (pl) -votre | your -vous | you - - | single letter forms - -c | c' -d | d' -j | j' -l | l' -à | to, at -m | m' -n | n' -s | s' -t | t' -y | there - - | forms of être (not including the infinitive): -été -étée -étées -étés -étant -suis -es -est -sommes -êtes -sont -serai -seras -sera -serons -serez -seront -serais -serait -serions -seriez -seraient -étais -était -étions -étiez -étaient -fus -fut -fûmes -fûtes -furent -sois -soit -soyons -soyez -soient -fusse -fusses -fût -fussions -fussiez -fussent - - | forms of avoir (not including the infinitive): -ayant -eu -eue -eues -eus -ai -as -avons -avez -ont -aurai -auras -aura -aurons -aurez -auront -aurais -aurait -aurions -auriez -auraient -avais -avait -avions -aviez -avaient -eut -eûmes -eûtes -eurent -aie -aies -ait -ayons -ayez -aient -eusse -eusses -eût -eussions -eussiez -eussent - - | Later additions (from Jean-Christophe Deschamps) -ceci | this -celà | that -cet | this -cette | this -ici | here -ils | they -les | the (pl) -leurs | their (pl) -quel | which -quels | which -quelle | which -quelles | which -sans | without -soi | oneself
\ No newline at end of file diff --git a/source/var/generate-top-words-list.sh b/source/var/generate-top-words-list.sh deleted file mode 100755 index 2a87e147..00000000 --- a/source/var/generate-top-words-list.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if test $# != 2 ; then - echo No input files given 1>&2 - exit 1 -fi - -awk 'BEGIN{FS = " "} { if ($1 ~ /^[A-Za-z]/) {print $1}}' $1 | sort | tr '\n' ';' > $2
\ No newline at end of file diff --git a/source/var/password b/source/var/password deleted file mode 100755 index be51ecd2..00000000 --- a/source/var/password +++ /dev/null @@ -1 +0,0 @@ -admin;d033e22ae348aeb5660fc2140aec35850c4da997 diff --git a/source/var/stop_words/english-stop-words-list.txt b/source/var/stop_words/english-stop-words-list.txt deleted file mode 100644 index caa26aaf..00000000 --- a/source/var/stop_words/english-stop-words-list.txt +++ /dev/null @@ -1 +0,0 @@ -a;about;above;after;again;against;all;am;an;and;any;are;aren't;as;at;be;because;been;before;being;below;between;both;but;by;cannot;can't;could;couldn't;did;didn't;do;does;doesn't;doing;don't;down;during;each;few;for;from;further;had;hadn't;has;hasn't;have;haven't;having;he;he'd;he'll;her;here;here's;hers;herself;he's;him;himself;his;how;how's;i;i'd;if;i'll;i'm;in;into;is;isn't;it;its;it's;itself;i've;let's;me;more;most;mustn't;my;myself;no;nor;not;of;off;on;once;only;or;other;ought;our;ours;ourselves;out;over;own;same;shan't;she;she'd;she'll;she's;should;shouldn't;slashdot;so;some;such;than;that;that's;the;their;theirs;them;themselves;then;there;there's;these;they;they'd;they'll;they're;they've;this;those;through;to;too;under;until;up;very;was;wasn't;we;we'd;we'll;were;we're;weren't;we've;what;what's;when;when's;where;where's;which;while;who;whom;who's;why;why's;with;won't;would;wouldn't;writes;you;you'd;you'll;your;you're;yours;yourself;yourselves;you've; diff --git a/source/var/stop_words/french-stop-words-list.txt b/source/var/stop_words/french-stop-words-list.txt deleted file mode 100644 index a6a36c79..00000000 --- a/source/var/stop_words/french-stop-words-list.txt +++ /dev/null @@ -1 +0,0 @@ -à;ai;aie;aient;aies;ait;as;au;aura;aurai;auraient;aurais;aurait;auras;aurez;auriez;aurions;aurons;auront;aux;avaient;avais;avait;avec;avez;aviez;avions;avons;ayant;ayez;ayons;c;ce;ceci;celà;ces;cet;cette;d;dans;de;des;du;elle;en;es;est;et;étaient;étais;était;étant;été;étée;étées;êtes;étés;étiez;étions;eu;eue;eues;eûmes;eurent;eus;eusse;eussent;eusses;eussiez;eussions;eut;eût;eûtes;eux;fûmes;furent;fus;fusse;fussent;fusses;fussiez;fussions;fut;fût;fûtes;ici;il;ils;j;je;l;la;le;les;leur;leurs;lui;m;ma;mais;me;même;mes;moi;mon;n;ne;nos;notre;nous;on;ont;ou;par;pas;pour;qu;que;quel;quelle;quelles;quels;qui;s;sa;sans;se;sera;serai;seraient;serais;serait;seras;serez;seriez;serions;serons;seront;ses;soi;soient;sois;soit;sommes;son;sont;soyez;soyons;suis;sur;t;ta;te;tes;toi;ton;tu;toujours;un;une;vos;votre;vous;y; |
