diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/web/forms.py | 8 | ||||
| -rwxr-xr-x | src/web/lib/misc_utils.py | 281 | ||||
| -rw-r--r-- | src/web/views/feed.py | 10 | ||||
| -rw-r--r-- | src/web/views/home.py | 4 | ||||
| -rw-r--r-- | src/web/views/user.py | 12 |
5 files changed, 298 insertions, 17 deletions
diff --git a/src/web/forms.py b/src/web/forms.py index fbfff85f..d1999b8b 100644 --- a/src/web/forms.py +++ b/src/web/forms.py @@ -35,7 +35,7 @@ from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \ SubmitField, IntegerField, SelectField, validators, HiddenField from flask.ext.wtf.html5 import EmailField -from web import utils +from web.lib import misc_utils from web.controllers import UserController from web.models import User @@ -76,12 +76,12 @@ class RedirectForm(Form): def __init__(self, *args, **kwargs): Form.__init__(self, *args, **kwargs) if not self.next.data: - self.next.data = utils.get_redirect_target() or '' + self.next.data = misc_utils.get_redirect_target() or '' def redirect(self, endpoint='home', **values): - if utils.is_safe_url(self.next.data): + if misc_utils.is_safe_url(self.next.data): return redirect(self.next.data) - target = utils.get_redirect_target() + target = misc_utils.get_redirect_target() return redirect(target or url_for(endpoint, **values)) diff --git a/src/web/lib/misc_utils.py b/src/web/lib/misc_utils.py new file mode 100755 index 00000000..92bdde8a --- /dev/null +++ b/src/web/lib/misc_utils.py @@ -0,0 +1,281 @@ +#! /usr/bin/env python +#-*- coding: utf-8 -*- + +# JARR - A Web based news aggregator. +# Copyright (C) 2010-2016 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://github.com/JARR-aggregator/JARR +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 1.8 $" +__date__ = "$Date: 2010/12/07 $" +__revision__ = "$Date: 2016/04/10 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "AGPLv3" + +# +# This file provides functions used for: +# - import from a JSON file; +# - generation of tags cloud; +# - HTML processing. +# + +import re +import sys +import glob +import opml +import json +import logging +import datetime +import operator +import urllib +import subprocess +import sqlalchemy +try: + from urlparse import urlparse, parse_qs, urlunparse +except: + from urllib.parse import urlparse, parse_qs, urlunparse, urljoin +from bs4 import BeautifulSoup +from collections import Counter +from contextlib import contextmanager +from flask import request + +import conf +from bootstrap import db +from web import controllers +from web.models import User, Feed, Article +from web.lib.utils import clear_string + +logger = logging.getLogger(__name__) + +ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) + +def is_safe_url(target): + """ + Ensures that a redirect target will lead to the same server. + """ + ref_url = urlparse(request.host_url) + test_url = urlparse(urljoin(request.host_url, target)) + return test_url.scheme in ('http', 'https') and \ + ref_url.netloc == test_url.netloc + +def get_redirect_target(): + """ + Looks at various hints to find the redirect target. + """ + for target in request.args.get('next'), request.referrer: + if not target: + continue + if is_safe_url(target): + return target + +def allowed_file(filename): + """ + Check if the uploaded file is allowed. + """ + return '.' in filename and \ + filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS + +@contextmanager +def opened_w_error(filename, mode="r"): + try: + f = open(filename, mode) + except IOError as err: + yield None, err + else: + try: + yield f, None + finally: + f.close() + +def fetch(id, feed_id=None): + """ + Fetch the feeds in a new processus. + The "asyncio" crawler is launched with the manager. + """ + cmd = [sys.executable, conf.BASE_DIR + '/manager.py', 'fetch_asyncio', + str(id), str(feed_id)] + return subprocess.Popen(cmd, stdout=subprocess.PIPE) + +def history(user_id, year=None, month=None): + """ + Sort articles by year and month. + """ + articles_counter = Counter() + articles = controllers.ArticleController(user_id).read() + if None != year: + articles = articles.filter(sqlalchemy.extract('year', Article.date) == year) + if None != month: + articles = articles.filter(sqlalchemy.extract('month', Article.date) == month) + for article in articles.all(): + if None != year: + articles_counter[article.date.month] += 1 + else: + articles_counter[article.date.year] += 1 + return articles_counter, articles + +def import_opml(email, opml_content): + """ + Import new feeds from an OPML file. + """ + user = User.query.filter(User.email == email).first() + try: + subscriptions = opml.from_string(opml_content) + except: + logger.exception("Parsing OPML file failed:") + raise + + def read(subsubscription, nb=0): + """ + Parse recursively through the categories and sub-categories. + """ + for subscription in subsubscription: + if len(subscription) != 0: + nb = read(subscription, nb) + else: + try: + title = subscription.text + except: + title = "" + try: + description = subscription.description + except: + description = "" + try: + link = subscription.xmlUrl + except: + continue + if None != Feed.query.filter(Feed.user_id == user.id, Feed.link == link).first(): + continue + try: + site_link = subscription.htmlUrl + except: + site_link = "" + new_feed = Feed(title=title, description=description, + link=link, site_link=site_link, + enabled=True) + user.feeds.append(new_feed) + nb += 1 + return nb + nb = read(subscriptions) + db.session.commit() + return nb + +def import_json(email, json_content): + """ + Import an account from a JSON file. + """ + user = User.query.filter(User.email == email).first() + json_account = json.loads(json_content) + nb_feeds, nb_articles = 0, 0 + # Create feeds: + for feed in json_account["result"]: + if None != Feed.query.filter(Feed.user_id == user.id, + Feed.link == feed["link"]).first(): + continue + new_feed = Feed(title=feed["title"], + description="", + link=feed["link"], + site_link=feed["site_link"], + created_date=datetime.datetime. + fromtimestamp(int(feed["created_date"])), + enabled=feed["enabled"]) + user.feeds.append(new_feed) + nb_feeds += 1 + db.session.commit() + # Create articles: + for feed in json_account["result"]: + user_feed = Feed.query.filter(Feed.user_id == user.id, + Feed.link == feed["link"]).first() + if None != user_feed: + for article in feed["articles"]: + if None == Article.query.filter(Article.user_id == user.id, + Article.feed_id == user_feed.id, + Article.link == article["link"]).first(): + new_article = Article(link=article["link"], + title=article["title"], + content=article["content"], + readed=article["readed"], + like=article["like"], + retrieved_date=datetime.datetime. + fromtimestamp(int(article["retrieved_date"])), + date=datetime.datetime. + fromtimestamp(int(article["date"])), + user_id=user.id, + feed_id=user_feed.id) + user_feed.articles.append(new_article) + nb_articles += 1 + db.session.commit() + return nb_feeds, nb_articles + +def clean_url(url): + """ + Remove utm_* parameters + """ + parsed_url = urlparse(url) + qd = parse_qs(parsed_url.query, keep_blank_values=True) + filtered = dict((k, v) for k, v in qd.items() + if not k.startswith('utm_')) + return urlunparse([ + parsed_url.scheme, + parsed_url.netloc, + urllib.parse.quote(urllib.parse.unquote(parsed_url.path)), + parsed_url.params, + urllib.parse.urlencode(filtered, doseq=True), + parsed_url.fragment + ]).rstrip('=') + +def load_stop_words(): + """ + Load the stop words and return them in a list. + """ + stop_words_lists = glob.glob('./JARR/var/stop_words/*.txt') + stop_words = [] + + for stop_wods_list in stop_words_lists: + with opened_w_error(stop_wods_list, "r") as (stop_wods_file, err): + if err: + stop_words = [] + else: + stop_words += stop_wods_file.read().split(";") + return stop_words + +def top_words(articles, n=10, size=5): + """ + Return the n most frequent words in a list. + """ + stop_words = load_stop_words() + words = Counter() + wordre = re.compile(r'\b\w{%s,}\b' % size, re.I) + for article in articles: + for word in [elem.lower() for elem in + wordre.findall(clear_string(article.content)) \ + if elem.lower() not in stop_words]: + words[word] += 1 + return words.most_common(n) + +def tag_cloud(tags): + """ + Generates a tags cloud. + """ + tags.sort(key=operator.itemgetter(0)) + return '\n'.join([('<font size=%d><a href="/search?query=%s" title="Count: %s">%s</a></font>' % \ + (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), word, format(count, ',d'), word)) \ + for (word, count) in tags]) + +if __name__ == "__main__": + import_opml("root@jarr.localhost", "./var/feeds_test.opml") + #import_opml("root@jarr.localhost", "./var/JARR.opml") diff --git a/src/web/views/feed.py b/src/web/views/feed.py index b6e0974a..b49495f3 100644 --- a/src/web/views/feed.py +++ b/src/web/views/feed.py @@ -10,7 +10,7 @@ from flask.ext.babel import gettext from flask.ext.login import login_required, current_user import conf -from web import utils +from web.lib import misc_utils, utils from web.lib.view_utils import etag_match from web.lib.feed_utils import construct_feed_from from web.forms import AddFeedForm @@ -47,8 +47,8 @@ def feed(feed_id=None): articles = ArticleController(current_user.id) \ .read(feed_id=feed_id) \ .order_by(desc("date")).all() - top_words = utils.top_words(articles, n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) + top_words = misc_utils.top_words(articles, n=50, size=int(word_size)) + tag_cloud = misc_utils.tag_cloud(top_words) today = datetime.now() try: @@ -126,7 +126,7 @@ def bookmarklet(): feed = feed_contr.create(**feed) flash(gettext('Feed was successfully created.'), 'success') if feed.enabled and conf.CRAWLING_METHOD == "classic": - utils.fetch(current_user.id, feed.id) + misc_utils.fetch(current_user.id, feed.id) flash(gettext("Downloading articles for the new feed..."), 'info') return redirect(url_for('feed.form', feed_id=feed.id)) @@ -215,7 +215,7 @@ def process_form(feed_id=None): feed_title=new_feed.title), 'success') if conf.CRAWLING_METHOD == "classic": - utils.fetch(current_user.id, new_feed.id) + misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), 'info') return redirect(url_for('feed.form', feed_id=new_feed.id)) diff --git a/src/web/views/home.py b/src/web/views/home.py index 12a06024..5a8312b4 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -8,7 +8,7 @@ from flask.ext.babel import gettext import conf from web.lib.utils import redirect_url -from web import utils +from web.lib import misc_utils from web.lib.view_utils import etag_match from web.models import Article from web.views.common import jsonify @@ -148,7 +148,7 @@ def fetch(feed_id=None): """ if conf.CRAWLING_METHOD == "classic" \ and (not conf.ON_HEROKU or current_user.is_admin): - utils.fetch(current_user.id, feed_id) + misc_utils.fetch(current_user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: flash(gettext("The manual retrieving of news is only available " + diff --git a/src/web/views/user.py b/src/web/views/user.py index 8568439f..3928f5dc 100644 --- a/src/web/views/user.py +++ b/src/web/views/user.py @@ -7,7 +7,7 @@ from flask.ext.login import login_required, current_user import conf from notifications import notifications -from web import utils +from web.lib import misc_utils from web.lib.user_utils import confirm_token from web.controllers import (UserController, FeedController, ArticleController, CategoryController) @@ -28,13 +28,13 @@ def management(): if None != request.files.get('opmlfile', None): # Import an OPML file data = request.files.get('opmlfile', None) - if not utils.allowed_file(data.filename): + if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: - nb = utils.import_opml(current_user.email, data.read()) + nb = misc_utils.import_opml(current_user.email, data.read()) if conf.CRAWLING_METHOD == "classic": - utils.fetch(current_user.email, None) + misc_utils.fetch(current_user.email, None) flash(str(nb) + ' ' + gettext('feeds imported.'), "success") flash(gettext("Downloading articles..."), 'info') @@ -44,11 +44,11 @@ def management(): elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) - if not utils.allowed_file(data.filename): + if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: - nb = utils.import_json(current_user.email, data.read()) + nb = misc_utils.import_json(current_user.email, data.read()) flash(gettext('Account imported.'), "success") except: flash(gettext("Impossible to import the account."), |