diff options
-rwxr-xr-x | fetch.py | 1 | ||||
-rwxr-xr-x | manager.py | 26 | ||||
-rw-r--r-- | pyaggr3g470r/crawler.py | 4 | ||||
-rwxr-xr-x | pyaggr3g470r/utils.py | 2 |
4 files changed, 29 insertions, 4 deletions
@@ -26,7 +26,6 @@ if __name__ == "__main__": feed_id = int(sys.argv[2]) except: feed_id = None - print(feed_id) for user in users: if user.activation_key == "": @@ -90,6 +90,32 @@ def fetch(user, password, limit=10): scheduler.run(limit=limit) scheduler.wait() +@manager.command +def fetch_old(user_id, feed_id): + "Uses the old crawler (temporary)" + with application.app_context(): + populate_g() + from pyaggr3g470r.models import User + from pyaggr3g470r import crawler + users, feed_id = [], None + try: + users = User.query.filter(User.id == int(user_id)).all() + except: + users = User.query.all() + finally: + if users == []: + users = User.query.all() + + try: + feed_id = int(feed_id) + except: + feed_id = None + + for user in users: + if user.activation_key == "": + print("Fetching articles for " + user.nickname) + feed_getter = crawler.retrieve_feed(user, feed_id) + if __name__ == '__main__': manager.run() diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index 5d2d2d4d..56c3552a 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -37,8 +37,8 @@ from bs4 import BeautifulSoup from sqlalchemy import or_ from pyaggr3g470r import utils -from pyaggr3g470r import conf -from pyaggr3g470r import db +import conf +from bootstrap import db from pyaggr3g470r.models import User, Article if not conf.ON_HEROKU: import pyaggr3g470r.search as fastsearch diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py index 1fc84ff4..a88c1a98 100755 --- a/pyaggr3g470r/utils.py +++ b/pyaggr3g470r/utils.py @@ -79,7 +79,7 @@ def opened_w_error(filename, mode="r"): f.close() def fetch(id, feed_id=None): - cmd = [conf.PYTHON, conf.basedir+'/fetch.py', str(id), str(feed_id)] + cmd = [conf.PYTHON, conf.basedir+'/manager.py', 'fetch_old', str(id), str(feed_id)] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) def import_opml(email, opml_content): |