aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2016-09-27 09:35:38 +0200
committerCédric Bonhomme <cedric@cedricbonhomme.org>2016-09-27 09:35:38 +0200
commit166e12b0d2c3350b28ed159b43a738cb09ebf1f6 (patch)
treed3b83827b9c01b0f93b354016e83d2939985aa28
parentUpdated dependencies. (diff)
downloadnewspipe-166e12b0d2c3350b28ed159b43a738cb09ebf1f6.tar.gz
newspipe-166e12b0d2c3350b28ed159b43a738cb09ebf1f6.tar.bz2
newspipe-166e12b0d2c3350b28ed159b43a738cb09ebf1f6.zip
Fixed a bug with SQLite support.
-rw-r--r--.gitmodules4
-rwxr-xr-xinstall.sh33
-rw-r--r--requirements.txt1
-rw-r--r--src/crawler/classic_crawler.py2
-rw-r--r--src/web/lib/article_utils.py16
5 files changed, 34 insertions, 22 deletions
diff --git a/.gitmodules b/.gitmodules
index fe23e1ee..91931bb4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
[submodule "submodules/bootstrap"]
- path = submodules/bootstrap
- url = https://github.com/twbs/bootstrap.git
+ path = submodules/bootstrap
+ url = https://github.com/twbs/bootstrap.git
diff --git a/install.sh b/install.sh
index 867dfc78..a0dc15ca 100755
--- a/install.sh
+++ b/install.sh
@@ -1,17 +1,17 @@
#! /usr/bin/env bash
#
-# This script install all dependencies and configure JARR
-# for Python 3.
+# This script install all dependencies and configure JARR for Python 3.
#
PYTHON_VERSION="3.5"
-sudo apt-get install -y python$PYTHON_VERSION libpq-dev python$PYTHON_VERSION-dev build-essential git
-sudo apt-get install -y libxml2-dev libxslt1-dev # for lxml
+#sudo apt-get install -y python$PYTHON_VERSION libpq-dev python$PYTHON_VERSION-dev build-essential git > /dev/null
+sudo apt-get install -y libxml2-dev libxslt1-dev > /dev/null # for lxml
-sed -i '/psycopg2/d' requirements.txt
-sudo pip$PYTHON_VERSION install --upgrade -r requirements.txt
+echo "Installing required Python libraries..."
+sed -i '/psycopg2/d' requirements.txt > /dev/null
+sudo pip$PYTHON_VERSION install --upgrade -r requirements.txt > /dev/null
# Initializes the configuration file
cp src/conf/conf.cfg-sample src/conf/conf.cfg
@@ -21,8 +21,10 @@ sed -i '/database/d' src/conf/conf.cfg
sed -i '/database_url/d' src/conf/conf.cfg
if [ "$1" == postgres ]; then
- sudo apt-get install -y postgresql postgresql-server-dev-9.4 postgresql-client
- sudo pip$PYTHON_VERSION install psycopg2
+ echo "Installing requirements for PostgreSQL..."
+ sudo apt-get install -y postgresql postgresql-server-dev-9.4 postgresql-client > /dev/null
+ sudo pip$PYTHON_VERSION install psycopg2 > /dev/null
+ echo "Configuring the database..."
echo "127.0.0.1:5433:aggregator:pgsqluser:pgsqlpwd" > ~/.pgpass
chmod 0600 ~/.pgpass
sudo -u postgres createuser pgsqluser --no-superuser --createdb --no-createrole
@@ -34,15 +36,22 @@ if [ "$1" == postgres ]; then
echo '[database]' >> src/conf/conf.cfg
echo 'database_url = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5433/aggregator' >> src/conf/conf.cfg
elif [ "$1" == sqlite ]; then
- sudo pip$PYTHON_VERSION install pysqlite # not working with Python 3!
# Add configuration lines for SQLite
+ echo "Configuring the SQLite database..."
echo '[database]' >> src/conf/conf.cfg
- echo 'database_url = sqlite+pysqlite:///jarr.db' >> src/conf/conf.cfg
+ echo 'database_url = sqlite:///jarr.db' >> src/conf/conf.cfg
fi
+echo "Initialization of the database..."
python$PYTHON_VERSION src/manager.py db_empty
python$PYTHON_VERSION src/manager.py db_create
# Bootstrap
-git submodule init
-git submodule update
+echo "Retrieving bootstrap..."
+git submodule init > /dev/null
+git submodule update > /dev/null
+
+
+echo "Installation terminated."
+echo "Launch JARR with the command:"
+echo -e "\tpython$PYTHON_VERSION src/runserver.py"
diff --git a/requirements.txt b/requirements.txt
index 818dc4c5..3128152f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,6 @@ feedparser==5.2.1
beautifulsoup4==4.5.1
lxml==3.6.4
opml==0.5
-psycopg2==2.6.2
SQLAlchemy==1.0.15
alembic==0.8.8
Flask==0.11.1
diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py
index cd395d0f..8b952940 100644
--- a/src/crawler/classic_crawler.py
+++ b/src/crawler/classic_crawler.py
@@ -148,6 +148,8 @@ async def insert_database(user, feed):
existing_article.dump())
continue
article = construct_article(article, feed)
+ print("test.............................................")
+ print(article)
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
diff --git a/src/web/lib/article_utils.py b/src/web/lib/article_utils.py
index 46bb9461..b5b9f246 100644
--- a/src/web/lib/article_utils.py
+++ b/src/web/lib/article_utils.py
@@ -36,21 +36,23 @@ def construct_article(entry, feed):
feed = feed.dump()
"Safe method to transorm a feedparser entry into an article"
now = datetime.now()
- date, updated_date = None, None
- for date_key in ('published', 'updated'):
+ date = None
+ for date_key in ('published', 'created', 'date'):
if entry.get(date_key):
try:
- date = dateutil.parser.parse(entry[date_key])
+ date = dateutil.parser.parse(entry[date_key])\
+ .astimezone(timezone.utc)
except Exception:
pass
else:
break
+
+ updated_date = None
try:
updated_date = dateutil.parser.parse(entry['updated'])
except Exception:
pass
content = get_article_content(entry)
-
article_link = entry.get('link')
return {'feed_id': feed['id'],
@@ -60,9 +62,9 @@ def construct_article(entry, feed):
'title': entry.get('title', 'No title'),
'readed': False, 'like': False,
'content': content,
- 'retrieved_date': now.isoformat(),
- 'date': (date or now).isoformat(),
- 'updated_date': (updated_date or date or now).isoformat()}
+ 'retrieved_date': now,
+ 'date': date or now,
+ 'updated_date': updated_date or date or now}
def get_article_content(entry):
content = ''
bgstack15