diff options
| -rw-r--r-- | .gitmodules | 4 | ||||
| -rwxr-xr-x | install.sh | 33 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | src/crawler/classic_crawler.py | 2 | ||||
| -rw-r--r-- | src/web/lib/article_utils.py | 16 |
5 files changed, 34 insertions, 22 deletions
diff --git a/.gitmodules b/.gitmodules index fe23e1ee..91931bb4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "submodules/bootstrap"] - path = submodules/bootstrap - url = https://github.com/twbs/bootstrap.git + path = submodules/bootstrap + url = https://github.com/twbs/bootstrap.git @@ -1,17 +1,17 @@ #! /usr/bin/env bash # -# This script install all dependencies and configure JARR -# for Python 3. +# This script install all dependencies and configure JARR for Python 3. # PYTHON_VERSION="3.5" -sudo apt-get install -y python$PYTHON_VERSION libpq-dev python$PYTHON_VERSION-dev build-essential git -sudo apt-get install -y libxml2-dev libxslt1-dev # for lxml +#sudo apt-get install -y python$PYTHON_VERSION libpq-dev python$PYTHON_VERSION-dev build-essential git > /dev/null +sudo apt-get install -y libxml2-dev libxslt1-dev > /dev/null # for lxml -sed -i '/psycopg2/d' requirements.txt -sudo pip$PYTHON_VERSION install --upgrade -r requirements.txt +echo "Installing required Python libraries..." +sed -i '/psycopg2/d' requirements.txt > /dev/null +sudo pip$PYTHON_VERSION install --upgrade -r requirements.txt > /dev/null # Initializes the configuration file cp src/conf/conf.cfg-sample src/conf/conf.cfg @@ -21,8 +21,10 @@ sed -i '/database/d' src/conf/conf.cfg sed -i '/database_url/d' src/conf/conf.cfg if [ "$1" == postgres ]; then - sudo apt-get install -y postgresql postgresql-server-dev-9.4 postgresql-client - sudo pip$PYTHON_VERSION install psycopg2 + echo "Installing requirements for PostgreSQL..." + sudo apt-get install -y postgresql postgresql-server-dev-9.4 postgresql-client > /dev/null + sudo pip$PYTHON_VERSION install psycopg2 > /dev/null + echo "Configuring the database..." echo "127.0.0.1:5433:aggregator:pgsqluser:pgsqlpwd" > ~/.pgpass chmod 0600 ~/.pgpass sudo -u postgres createuser pgsqluser --no-superuser --createdb --no-createrole @@ -34,15 +36,22 @@ if [ "$1" == postgres ]; then echo '[database]' >> src/conf/conf.cfg echo 'database_url = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5433/aggregator' >> src/conf/conf.cfg elif [ "$1" == sqlite ]; then - sudo pip$PYTHON_VERSION install pysqlite # not working with Python 3! # Add configuration lines for SQLite + echo "Configuring the SQLite database..." echo '[database]' >> src/conf/conf.cfg - echo 'database_url = sqlite+pysqlite:///jarr.db' >> src/conf/conf.cfg + echo 'database_url = sqlite:///jarr.db' >> src/conf/conf.cfg fi +echo "Initialization of the database..." python$PYTHON_VERSION src/manager.py db_empty python$PYTHON_VERSION src/manager.py db_create # Bootstrap -git submodule init -git submodule update +echo "Retrieving bootstrap..." +git submodule init > /dev/null +git submodule update > /dev/null + + +echo "Installation terminated." +echo "Launch JARR with the command:" +echo -e "\tpython$PYTHON_VERSION src/runserver.py" diff --git a/requirements.txt b/requirements.txt index 818dc4c5..3128152f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ feedparser==5.2.1 beautifulsoup4==4.5.1 lxml==3.6.4 opml==0.5 -psycopg2==2.6.2 SQLAlchemy==1.0.15 alembic==0.8.8 Flask==0.11.1 diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py index cd395d0f..8b952940 100644 --- a/src/crawler/classic_crawler.py +++ b/src/crawler/classic_crawler.py @@ -148,6 +148,8 @@ async def insert_database(user, feed): existing_article.dump()) continue article = construct_article(article, feed) + print("test.............................................") + print(article) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", diff --git a/src/web/lib/article_utils.py b/src/web/lib/article_utils.py index 46bb9461..b5b9f246 100644 --- a/src/web/lib/article_utils.py +++ b/src/web/lib/article_utils.py @@ -36,21 +36,23 @@ def construct_article(entry, feed): feed = feed.dump() "Safe method to transorm a feedparser entry into an article" now = datetime.now() - date, updated_date = None, None - for date_key in ('published', 'updated'): + date = None + for date_key in ('published', 'created', 'date'): if entry.get(date_key): try: - date = dateutil.parser.parse(entry[date_key]) + date = dateutil.parser.parse(entry[date_key])\ + .astimezone(timezone.utc) except Exception: pass else: break + + updated_date = None try: updated_date = dateutil.parser.parse(entry['updated']) except Exception: pass content = get_article_content(entry) - article_link = entry.get('link') return {'feed_id': feed['id'], @@ -60,9 +62,9 @@ def construct_article(entry, feed): 'title': entry.get('title', 'No title'), 'readed': False, 'like': False, 'content': content, - 'retrieved_date': now.isoformat(), - 'date': (date or now).isoformat(), - 'updated_date': (updated_date or date or now).isoformat()} + 'retrieved_date': now, + 'date': date or now, + 'updated_date': updated_date or date or now} def get_article_content(entry): content = '' |