aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2016-02-02 23:30:57 +0100
committerFrançois Schmidts <francois.schmidts@gmail.com>2016-02-02 23:30:57 +0100
commit4a8438d7f2b7b16941240b91f39a9402c431ffc2 (patch)
tree89380f41b802256d8fdbf724e7d9e63b48209b4a /src
parentMerge branch 'feature/categories' (diff)
downloadnewspipe-4a8438d7f2b7b16941240b91f39a9402c431ffc2.tar.gz
newspipe-4a8438d7f2b7b16941240b91f39a9402c431ffc2.tar.bz2
newspipe-4a8438d7f2b7b16941240b91f39a9402c431ffc2.zip
writing a bit of doc, moving crawler together
Diffstat (limited to 'src')
-rw-r--r--src/crawler/classic_crawler.py (renamed from src/crawler.py)0
-rw-r--r--src/crawler/http_crawler.py (renamed from src/web/lib/crawler.py)0
-rwxr-xr-xsrc/manager.py6
-rwxr-xr-xsrc/web/utils.py4
4 files changed, 5 insertions, 5 deletions
diff --git a/src/crawler.py b/src/crawler/classic_crawler.py
index 0598c418..0598c418 100644
--- a/src/crawler.py
+++ b/src/crawler/classic_crawler.py
diff --git a/src/web/lib/crawler.py b/src/crawler/http_crawler.py
index f480fe96..f480fe96 100644
--- a/src/web/lib/crawler.py
+++ b/src/crawler/http_crawler.py
diff --git a/src/manager.py b/src/manager.py
index f7240670..781d742b 100755
--- a/src/manager.py
+++ b/src/manager.py
@@ -32,7 +32,7 @@ def db_create():
@manager.command
def fetch(limit=100, retreive_all=False):
"Crawl the feeds with the client crawler."
- from web.lib.crawler import CrawlerScheduler
+ from crawler.http_crawler import CrawlerScheduler
scheduler = CrawlerScheduler(conf.API_LOGIN, conf.API_PASSWD)
scheduler.run(limit=limit, retreive_all=retreive_all)
scheduler.wait()
@@ -47,7 +47,7 @@ def fetch_asyncio(user_id, feed_id):
populate_g()
from flask import g
from web.models import User
- import crawler
+ from crawler import classic_crawler
users = []
try:
users = User.query.filter(User.id == int(user_id)).all()
@@ -67,7 +67,7 @@ def fetch_asyncio(user_id, feed_id):
if user.activation_key == "":
print("Fetching articles for " + user.nickname)
g.user = user
- crawler.retrieve_feed(loop, g.user, feed_id)
+ classic_crawler.retrieve_feed(loop, g.user, feed_id)
loop.close()
from scripts.probes import ArticleProbe, FeedProbe
diff --git a/src/web/utils.py b/src/web/utils.py
index fcd791e8..1d4b30ab 100755
--- a/src/web/utils.py
+++ b/src/web/utils.py
@@ -109,8 +109,8 @@ def fetch(id, feed_id=None):
Fetch the feeds in a new processus.
The "asyncio" crawler is launched with the manager.
"""
- cmd = [sys.executable, conf.BASE_DIR+'/manager.py', 'fetch_asyncio', str(id),
- str(feed_id)]
+ cmd = [sys.executable, conf.BASE_DIR + '/manager.py', 'fetch_asyncio',
+ str(id), str(feed_id)]
return subprocess.Popen(cmd, stdout=subprocess.PIPE)
def history(user_id, year=None, month=None):
bgstack15