aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bootstrap.py4
-rw-r--r--documentation/deployment.rst3
-rw-r--r--documentation/web-services.rst72
-rw-r--r--pyaggr3g470r/decorators.py13
-rwxr-xr-xpyaggr3g470r/lib/client.py16
-rw-r--r--pyaggr3g470r/lib/crawler.py7
-rw-r--r--pyaggr3g470r/lib/exceptions.py13
-rw-r--r--pyaggr3g470r/search.py2
-rw-r--r--pyaggr3g470r/static/js/articles.js8
-rw-r--r--pyaggr3g470r/templates/layout.html10
-rw-r--r--pyaggr3g470r/views/api/common.py33
-rw-r--r--pyaggr3g470r/views/article.py2
-rw-r--r--pyaggr3g470r/views/feed.py6
-rw-r--r--requirements.txt2
-rwxr-xr-xrunserver.py2
15 files changed, 75 insertions, 118 deletions
diff --git a/bootstrap.py b/bootstrap.py
index 5d599146..83862640 100644
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -21,14 +21,14 @@ def set_logging(log_path, log_level=logging.DEBUG,
logger.addHandler(handler)
logger.setLevel(log_level)
-set_logging(conf.LOG_PATH)
-
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
# Create Flask application
application = Flask('pyaggr3g470r')
application.debug = conf.WEBSERVER_DEBUG
+set_logging(conf.LOG_PATH, log_level=logging.DEBUG if conf.WEBSERVER_DEBUG
+ else logging.INFO)
# Create dummy secrey key so we can use sessions
application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None)
diff --git a/documentation/deployment.rst b/documentation/deployment.rst
index d0639c45..d06d55fe 100644
--- a/documentation/deployment.rst
+++ b/documentation/deployment.rst
@@ -92,6 +92,7 @@ If you want to use PostgreSQL
.. code-block:: bash
$ sudo apt-get install postgresql postgresql-server-dev-9.3 postgresql-client
+ $ pip install psycopg2
$ echo "127.0.0.1:5432:aggregator:pgsqluser:pgsqlpwd" > ~/.pgpass
$ chmod 700 ~/.pgpass
$ sudo -u postgres createuser pgsqluser --no-superuser --createdb --no-createrole
@@ -131,7 +132,7 @@ Configuration
=============
Configuration (database url, email, proxy, user agent, etc.) is done via the file *conf/conf.cfg*.
-Check these configuration before executing *db_create.py*.
+Check these configuration before executing *db_create.py*.
If you want to use pyAggr3g470r with Tor/Privoxy, you just have to set the value of
*http_proxy* (most of the time: *http_proxy = 127.0.0.1:8118**). Else leave the value blank.
diff --git a/documentation/web-services.rst b/documentation/web-services.rst
index 2d724569..dd2d0125 100644
--- a/documentation/web-services.rst
+++ b/documentation/web-services.rst
@@ -7,10 +7,11 @@ Articles
.. code-block:: python
>>> import requests, json
- >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", auth=("your-email", "your-password"))
- >>> r.status_code-block
- 200
- >>> rjson = json.loads(r.text)
+ >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles",
+ ... auth=("your-nickname", "your-password"))
+ >>> r.status_code
+ 200 # OK
+ >>> rjson = r.json()
>>> rjson["result"][0]["title"]
u'Sponsors required for KDE code sprint in Randa'
>>> rjson["result"][0]["date"]
@@ -20,15 +21,19 @@ Possible parameters:
.. code-block:: bash
- $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=unread&feed=24"
- $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=read&feed=24&limit=20"
- $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=all&feed=24&limit=20"
+ $ curl --user your-nickname:your-password "https://pyaggr3g470r.herokuapp.com/api/v2.0/articles" -H 'Content-Type: application/json' --data='{"feed": 24}'
-Get an article:
+Get an article with another way to pass credentials :
.. code-block:: bash
- $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84566"
+ $ curl "https://your-nickname:your-password@pyaggr3g470r.herokuapp.com/api/v2.0/article/84566"
+
+And delete it :
+
+.. code-block:: bash
+
+ $ curl -XDELETE "https://your-nickname:your-password@pyaggr3g470r.herokuapp.com/api/v2.0/article/84566"
Add an article:
@@ -36,13 +41,18 @@ Add an article:
>>> import requests, json
>>> headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
- >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/', 'title': 'Sortie de pyAggr3g470r 5.3', 'content':'La page principale de pyAggr3g470r a été améliorée...', 'date':'06/23/2014 11:42 AM', 'feed_id':'42'}
- >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload))
- >>> print r.content
- {
- "message": "ok"
- }
- >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?feed=42&limit=1", auth=("your-email", "your-password"))
+ >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/',
+ ... 'title': 'Sortie de pyAggr3g470r 5.3',
+ ... 'content':'La page principale de pyAggr3g470r a été améliorée...',
+ ... 'date':'2014/06/23T11:42:20 GMT',
+ ... 'feed_id':'42'}
+ >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/article",
+ ... headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload))
+ >>> print r.status_code
+ 201 # Created
+ >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles",
+ ... auth=("your-nickname", "your-password")
+ ... data=json.dumps({'feed_id': 42, 'limit': 1}))
>>> print json.loads(r.content)["result"][0]["title"]
Sortie de pyAggr3g470r 5.3
@@ -51,30 +61,20 @@ Update an article:
.. code-block:: python
>>> payload = {"like":True, "readed":False}
- >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/65", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload))
- >>> print r.content
- {
- "message": "ok"
- }
+ >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/65", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload))
+ >>> print r.status_code
+ 200 # OK
Delete an article:
.. code-block:: python
- >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password"))
+ >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", auth=("your-nickname", "your-password"))
>>> print r.status_code
- 200
- >>> print r.content
- {
- "message": "ok"
- }
- >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password"))
+ 204 # deleted - No content
+ >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", auth=("your-nickname", "your-password"))
>>> print r.status_code
- 200
- >>> print r.content
- {
- "message": "Article not found."
- }
+ 404 # not found
Feeds
-----
@@ -84,17 +84,17 @@ Add a feed:
.. code-block:: python
>>> payload = {'link': 'http://blog.cedricbonhomme.org/feed'}
- >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload))
+ >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload))
Update a feed:
.. code-block:: python
>>> payload = {"title":"Feed new title", "description":"New description"}
- >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/42", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload))
+ >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/42", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload))
Delete a feed:
.. code-block:: python
- >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/29", auth=("your-email", "your-password"))
+ >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/29", auth=("your-nickname", "your-password"))
diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py
index 9bae626d..9e8f9c0b 100644
--- a/pyaggr3g470r/decorators.py
+++ b/pyaggr3g470r/decorators.py
@@ -9,7 +9,6 @@ from flask.ext.babel import gettext
from flask.ext.login import login_required
from pyaggr3g470r.models import Feed
-from pyaggr3g470r.lib.exceptions import PyAggError
def async(f):
@@ -43,20 +42,8 @@ def feed_access_required(func):
return decorated
-def handle_pyagg_error(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- try:
- return func(*args, **kwargs)
- except PyAggError as error:
- flash(gettext(error.default_message), 'warning')
- return redirect(url_for('home'))
- return wrapper
-
-
def pyagg_default_decorator(func):
@login_required
- @handle_pyagg_error
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py
deleted file mode 100755
index 6b2fc9ae..00000000
--- a/pyaggr3g470r/lib/client.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-import json
-import requests
-import conf
-
-
-def get_client(email, password):
- client = requests.session()
- client.get(conf.PLATFORM_URL + 'api/csrf', verify=False,
- data=json.dumps({'email': email,
- 'password': password}))
- return client
-
-
-def get_articles(client):
- return client.get(conf.PLATFORM_URL + 'api/v1.0/articles/').json
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index 6697e4c3..de770934 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -10,6 +10,7 @@ from requests_futures.sessions import FuturesSession
from pyaggr3g470r.lib.utils import default_handler
logger = logging.getLogger(__name__)
+API_ROOT = "api/v2.0/"
def extract_id(entry, keys=[('link', 'link'),
@@ -52,7 +53,7 @@ class AbstractCrawler:
if data is None:
data = {}
method = getattr(self.session, method)
- return method("%sapi/v1.0/%s" % (self.url, urn),
+ return method("%s%s%s" % (self.url, API_ROOT, urn),
auth=self.auth, data=json.dumps(data,
default=default_handler),
headers={'Content-Type': 'application/json'})
@@ -193,7 +194,7 @@ class CrawlerScheduler(AbstractCrawler):
headers=self.prepare_headers(feed))
future.add_done_callback(FeedCrawler(feed, self.auth).callback)
- def run(self):
+ def run(self, **kwargs):
logger.debug('retreving fetchable feed')
- future = self.query_pyagg('get', 'feeds/fetchable')
+ future = self.query_pyagg('get', 'feeds/fetchable', kwargs)
future.add_done_callback(self.callback)
diff --git a/pyaggr3g470r/lib/exceptions.py b/pyaggr3g470r/lib/exceptions.py
deleted file mode 100644
index 30c71a5c..00000000
--- a/pyaggr3g470r/lib/exceptions.py
+++ /dev/null
@@ -1,13 +0,0 @@
-class PyAggError(Exception):
- status_code = None
- default_message = ''
-
-
-class Forbidden(PyAggError):
- status_code = 403
- default_message = 'You do not have the rights to access that resource'
-
-
-class NotFound(PyAggError):
- status_code = 404
- default_message = 'Resource was not found'
diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py
index 89fa0860..a7f780df 100644
--- a/pyaggr3g470r/search.py
+++ b/pyaggr3g470r/search.py
@@ -102,7 +102,7 @@ def delete_article(user_id, feed_id, article_id):
try:
ix = open_dir(indexdir)
except (EmptyIndexError, OSError):
- raise EmptyIndexError
+ return
writer = ix.writer()
document = And([Term("user_id", user_id), Term("feed_id", feed_id),
Term("article_id", article_id)])
diff --git a/pyaggr3g470r/static/js/articles.js b/pyaggr3g470r/static/js/articles.js
index 51273f3d..312a5cb6 100644
--- a/pyaggr3g470r/static/js/articles.js
+++ b/pyaggr3g470r/static/js/articles.js
@@ -18,6 +18,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+API_ROOT = 'api/v2.0/'
+
if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
+function ($) {
@@ -78,7 +80,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// Encode your data as JSON.
data: data,
// This is the type of data you're expecting back from the server.
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
@@ -114,7 +116,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// Encode your data as JSON.
data: data,
// This is the type of data you're expecting back from the server.
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
@@ -132,7 +134,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// sends the updates to the server
$.ajax({
type: 'DELETE',
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html
index 4dc62350..6b929bf3 100644
--- a/pyaggr3g470r/templates/layout.html
+++ b/pyaggr3g470r/templates/layout.html
@@ -7,9 +7,9 @@
<meta name="description" content="pyAggr3g470r is a web-based news aggregator." />
<meta name="author" content="" />
<title>{% if head_title %}{{ head_title }} - {% endif %}pyAggr3g470r</title>
- <link rel="shortcut icon" href="{{ url_for('.static', filename='img/favicon.png') }}" />
+ <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.png') }}" />
<!-- Bootstrap core CSS -->
- <link href="{{ url_for('.static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
+ <link href="{{ url_for('static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
<!-- Add custom CSS here -->
<style>
body {
@@ -155,9 +155,9 @@
<!-- Bootstrap core JavaScript -->
<!-- Placed at the end of the document so the pages load faster -->
- <script src="{{ url_for('.static', filename = 'js/jquery.js') }}"></script>
- <script src="{{ url_for('.static', filename = 'js/bootstrap.js') }}"></script>
- <script src="{{ url_for('.static', filename = 'js/articles.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/jquery.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/bootstrap.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/articles.js') }}"></script>
<script type="text/javascript" class="source">
if (window.location.href.indexOf("filter_=all") > -1){
$("#tab-all").attr('class', "active");
diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py
index a9d35411..a136645c 100644
--- a/pyaggr3g470r/views/api/common.py
+++ b/pyaggr3g470r/views/api/common.py
@@ -2,12 +2,12 @@ import json
import logging
import dateutil.parser
from functools import wraps
+from werkzeug.exceptions import Unauthorized
from flask import request, g, session, Response
from flask.ext.restful import Resource, reqparse
from pyaggr3g470r.lib.utils import default_handler
from pyaggr3g470r.models import User
-from pyaggr3g470r.lib.exceptions import PyAggError
logger = logging.getLogger(__name__)
@@ -18,36 +18,31 @@ def authenticate(func):
"""
@wraps(func)
def wrapper(*args, **kwargs):
+ logged_in = False
if not getattr(func, 'authenticated', True):
- return func(*args, **kwargs)
-
+ logged_in = True
# authentication based on the session (already logged on the site)
- if 'email' in session or g.user.is_authenticated():
- return func(*args, **kwargs)
-
- # authentication via HTTP only
- auth = request.authorization
- try:
+ elif 'email' in session or g.user.is_authenticated():
+ logged_in = True
+ else:
+ # authentication via HTTP only
+ auth = request.authorization
user = User.query.filter(User.nickname == auth.username).first()
if user and user.check_password(auth.password) \
and user.activation_key == "":
g.user = user
- except Exception:
- return Response('<Authentication required>', 401,
- {'WWWAuthenticate':
- 'Basic realm="Login Required"'})
- return func(*args, **kwargs)
+ logged_in = True
+
+ if logged_in:
+ return func(*args, **kwargs)
+ raise Unauthorized({'WWWAuthenticate': 'Basic realm="Login Required"'})
return wrapper
def to_response(func):
def wrapper(*args, **kwargs):
status_code = 200
- try:
- result = func(*args, **kwargs)
- except PyAggError as error:
- return Response(json.dumps(error, default=default_handler),
- status=status_code)
+ result = func(*args, **kwargs)
if isinstance(result, Response):
return result
elif isinstance(result, tuple):
diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py
index 21858a33..66cc0f37 100644
--- a/pyaggr3g470r/views/article.py
+++ b/pyaggr3g470r/views/article.py
@@ -21,7 +21,7 @@ def articles(feed_id=None, nb_articles=-1):
if len(feed.articles.all()) <= nb_articles:
nb_articles = -1
if nb_articles == -1:
- feed.articles = feed.article.limit(nb_articles)
+ feed.articles = feed.articles.limit(nb_articles)
return render_template('articles.html', feed=feed, nb_articles=nb_articles)
diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py
index fa9dc23d..2af502a7 100644
--- a/pyaggr3g470r/views/feed.py
+++ b/pyaggr3g470r/views/feed.py
@@ -28,15 +28,15 @@ def feed(feed_id=None):
top_words = utils.top_words(articles, n=50, size=int(word_size))
tag_cloud = utils.tag_cloud(top_words)
- today = datetime.datetime.now()
+ today = datetime.now()
try:
last_article = articles[0].date
first_article = articles[-1].date
delta = last_article - first_article
average = round(float(len(articles)) / abs(delta.days), 2)
except:
- last_article = datetime.datetime.fromtimestamp(0)
- first_article = datetime.datetime.fromtimestamp(0)
+ last_article = datetime.fromtimestamp(0)
+ first_article = datetime.fromtimestamp(0)
delta = last_article - first_article
average = 0
elapsed = today - last_article
diff --git a/requirements.txt b/requirements.txt
index 60869a05..c988f7dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,6 @@ requests
beautifulsoup4
lxml
SQLAlchemy
-psycopg2
Flask
Flask-SQLAlchemy
Flask-Login
@@ -21,3 +20,4 @@ python-postmark
whoosh
python-dateutil
alembic
+requests-futures==0.9.5
diff --git a/runserver.py b/runserver.py
index 8d163cd6..5a4aa1fe 100755
--- a/runserver.py
+++ b/runserver.py
@@ -45,7 +45,7 @@ from flask.ext.restful import Api
from flask import g
with application.app_context():
- g.api = Api(application, prefix='/api/v1.0')
+ g.api = Api(application, prefix='/api/v2.0')
g.babel = babel
g.allowed_file = allowed_file
g.db = db
bgstack15