aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2015-03-04 08:30:08 +0100
committerCédric Bonhomme <cedric@cedricbonhomme.org>2015-03-04 08:30:08 +0100
commita6d1c3fd860cff01510cd63264cb672c677a2e69 (patch)
tree702fb769db8532c64c0a5ac7132105329b053506
parentThe feed abject should be loaded when 'feed_id' is not None. (diff)
downloadnewspipe-a6d1c3fd860cff01510cd63264cb672c677a2e69.tar.gz
newspipe-a6d1c3fd860cff01510cd63264cb672c677a2e69.tar.bz2
newspipe-a6d1c3fd860cff01510cd63264cb672c677a2e69.zip
Test with the old crawler (temporary during the transition).
-rwxr-xr-xfetch.py1
-rwxr-xr-xmanager.py26
-rw-r--r--pyaggr3g470r/crawler.py4
-rwxr-xr-xpyaggr3g470r/utils.py2
4 files changed, 29 insertions, 4 deletions
diff --git a/fetch.py b/fetch.py
index ce4adb50..db950aa0 100755
--- a/fetch.py
+++ b/fetch.py
@@ -26,7 +26,6 @@ if __name__ == "__main__":
feed_id = int(sys.argv[2])
except:
feed_id = None
- print(feed_id)
for user in users:
if user.activation_key == "":
diff --git a/manager.py b/manager.py
index 89fd2bf1..c139d93b 100755
--- a/manager.py
+++ b/manager.py
@@ -90,6 +90,32 @@ def fetch(user, password, limit=10):
scheduler.run(limit=limit)
scheduler.wait()
+@manager.command
+def fetch_old(user_id, feed_id):
+ "Uses the old crawler (temporary)"
+ with application.app_context():
+ populate_g()
+ from pyaggr3g470r.models import User
+ from pyaggr3g470r import crawler
+ users, feed_id = [], None
+ try:
+ users = User.query.filter(User.id == int(user_id)).all()
+ except:
+ users = User.query.all()
+ finally:
+ if users == []:
+ users = User.query.all()
+
+ try:
+ feed_id = int(feed_id)
+ except:
+ feed_id = None
+
+ for user in users:
+ if user.activation_key == "":
+ print("Fetching articles for " + user.nickname)
+ feed_getter = crawler.retrieve_feed(user, feed_id)
+
if __name__ == '__main__':
manager.run()
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index 5d2d2d4d..56c3552a 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -37,8 +37,8 @@ from bs4 import BeautifulSoup
from sqlalchemy import or_
from pyaggr3g470r import utils
-from pyaggr3g470r import conf
-from pyaggr3g470r import db
+import conf
+from bootstrap import db
from pyaggr3g470r.models import User, Article
if not conf.ON_HEROKU:
import pyaggr3g470r.search as fastsearch
diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py
index 1fc84ff4..a88c1a98 100755
--- a/pyaggr3g470r/utils.py
+++ b/pyaggr3g470r/utils.py
@@ -79,7 +79,7 @@ def opened_w_error(filename, mode="r"):
f.close()
def fetch(id, feed_id=None):
- cmd = [conf.PYTHON, conf.basedir+'/fetch.py', str(id), str(feed_id)]
+ cmd = [conf.PYTHON, conf.basedir+'/manager.py', 'fetch_old', str(id), str(feed_id)]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
def import_opml(email, opml_content):
bgstack15