From e28458a74f28fa9eeed374cc851497deaf90b88d Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 29 Aug 2022 18:25:29 -0400 Subject: Backend: support "folder" POST param and add config options --- app/main.py | 7 +++++-- app/ytdl.py | 33 +++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index 9fbdf19..ecc5040 100644 --- a/app/main.py +++ b/app/main.py @@ -16,11 +16,13 @@ class Config: _DEFAULTS = { 'DOWNLOAD_DIR': '.', 'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR', + 'CUSTOM_DIR': 'true', + 'AUTO_CREATE_CUSTOM_DIR': 'false', 'STATE_DIR': '.', 'URL_PREFIX': '', 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', - 'YTDL_OPTIONS': '{}', + 'YTDL_OPTIONS': '{}' } def __init__(self): @@ -80,7 +82,8 @@ async def add(request): if not url or not quality: raise web.HTTPBadRequest() format = post.get('format') - status = await dqueue.add(url, quality, format) + folder = post.get('folder') + status = await dqueue.add(url, quality, format, folder) return web.Response(text=serializer.encode(status)) @routes.post(config.URL_PREFIX + 'delete') diff --git a/app/ytdl.py b/app/ytdl.py index 21b82da..29fa4bd 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -27,10 +27,11 @@ class DownloadQueueNotifier: raise NotImplementedError class DownloadInfo: - def __init__(self, id, title, url, quality, format): + def __init__(self, id, title, url, quality, format, folder): self.id, self.title, self.url = id, title, url self.quality = quality self.format = format + self.folder = folder self.status = self.msg = self.percent = self.speed = self.eta = None self.filename = None self.timestamp = time.time_ns() @@ -192,7 +193,7 @@ class DownloadQueue: async def __import_queue(self): for k, v in self.queue.saved_items(): - await self.add(v.url, v.quality, v.format) + await self.add(v.url, v.quality, v.format, folder=v.folder) async def initialize(self): self.event = asyncio.Event() @@ -207,7 +208,7 @@ class DownloadQueue: **self.config.YTDL_OPTIONS, }).extract_info(url, download=False) - async def __add_entry(self, entry, quality, format, already): + async def __add_entry(self, entry, quality, format, already, folder=None): etype = entry.get('_type') or 'video' if etype == 'playlist': entries = entry['entries'] @@ -220,14 +221,26 @@ class DownloadQueue: for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, already)) + results.append(await self.__add_entry(etr, quality, format, already, folder=folder)) if any(res['status'] == 'error' for res in results): return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry: if not self.queue.exists(entry['id']): - dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format) - dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR + dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder) + base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR + if folder: + if self.config.CUSTOM_DIR != 'true': + return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIR is not true in the configuration.'} + dldirectory = os.path.realpath(os.path.join(base_directory, folder)) + if not dldirectory.startswith(base_directory): + return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{base_directory}"'} + if not os.path.isdir(dldirectory): + if self.config.AUTO_CREATE_CUSTOM_DIR != 'true': + return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist, and AUTO_CREATE_CUSTOM_DIR is not true in the configuration.'} + os.makedirs(dldirectory, exist_ok=True) + else: + dldirectory = base_directory output = self.config.OUTPUT_TEMPLATE output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER for property, value in entry.items(): @@ -238,11 +251,11 @@ class DownloadQueue: await self.notifier.added(dl) return {'status': 'ok'} elif etype.startswith('url'): - return await self.add(entry['url'], quality, format, already) + return await self.add(entry['url'], quality, format, already, folder=folder) return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - async def add(self, url, quality, format, already=None): - log.info(f'adding {url}') + async def add(self, url, quality, format, already=None, folder=None): + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}') already = set() if already is None else already if url in already: log.info('recursion detected, skipping') @@ -253,7 +266,7 @@ class DownloadQueue: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, already) + return await self.__add_entry(entry, quality, format, already, folder=folder) async def cancel(self, ids): for id in ids: -- cgit From 8857878ec2fe6712de46347b5a2d65c64b94ee28 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 29 Aug 2022 19:01:50 -0400 Subject: show error if static assets are not found --- app/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index ecc5040..dcacfec 100644 --- a/app/main.py +++ b/app/main.py @@ -116,7 +116,13 @@ if config.URL_PREFIX != '/': routes.static(config.URL_PREFIX + 'favicon/', 'favicon') routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR) routes.static(config.URL_PREFIX, 'ui/dist/metube') -app.add_routes(routes) +try: + app.add_routes(routes) +except ValueError as e: + if 'ui/dist/metube' in str(e): + raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build`') from e + raise e + # https://github.com/aio-libs/aiohttp/pull/4615 waiting for release -- cgit From 4a9f55adda55a35c67c5e6699aa71fa56295c9b4 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 29 Aug 2022 20:27:34 -0400 Subject: Propagate configuration on load via downloads socket --- app/main.py | 1 + 1 file changed, 1 insertion(+) (limited to 'app') diff --git a/app/main.py b/app/main.py index dcacfec..bbdfc87 100644 --- a/app/main.py +++ b/app/main.py @@ -99,6 +99,7 @@ async def delete(request): @sio.event async def connect(sid, environ): await sio.emit('all', serializer.encode(dqueue.get()), to=sid) + await sio.emit('configuration', serializer.encode(config), to=sid) @routes.get(config.URL_PREFIX) def index(request): -- cgit From f79c8fa7542822cd3edd542d646fc5881d3bb80e Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 29 Aug 2022 20:41:21 -0400 Subject: pass custom_directories from server to client --- app/main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index bbdfc87..56adc00 100644 --- a/app/main.py +++ b/app/main.py @@ -7,6 +7,7 @@ from aiohttp import web import socketio import logging import json +import pathlib from ytdl import DownloadQueueNotifier, DownloadQueue @@ -100,6 +101,18 @@ async def delete(request): async def connect(sid, environ): await sio.emit('all', serializer.encode(dqueue.get()), to=sid) await sio.emit('configuration', serializer.encode(config), to=sid) + if config.CUSTOM_DIR: + await sio.emit('custom_directories', serializer.encode(get_custom_directories()), to=sid) + +def get_custom_directories(): + path = pathlib.Path(config.DOWNLOAD_DIR) + # Recursively lists all subdirectories, and converts PosixPath objects to string + dirs = list(map(str, path.glob('**'))) + + if '.' in dirs: + dirs.remove('.') + + return {"directories": dirs} @routes.get(config.URL_PREFIX) def index(request): @@ -121,7 +134,7 @@ try: app.add_routes(routes) except ValueError as e: if 'ui/dist/metube' in str(e): - raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build`') from e + raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e raise e -- cgit From ba712fc071398e615ead822c8bd81aad42a90c8f Mon Sep 17 00:00:00 2001 From: James Woglom Date: Tue, 30 Aug 2022 00:55:16 -0400 Subject: Fill in download_dir or audio_download_dir on launch --- app/main.py | 43 ++++++++++++++++++++++++++++++++----------- app/ytdl.py | 9 +++++---- 2 files changed, 37 insertions(+), 15 deletions(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index 56adc00..de13f1e 100644 --- a/app/main.py +++ b/app/main.py @@ -17,8 +17,8 @@ class Config: _DEFAULTS = { 'DOWNLOAD_DIR': '.', 'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR', - 'CUSTOM_DIR': 'true', - 'AUTO_CREATE_CUSTOM_DIR': 'false', + 'CUSTOM_DIRS': 'true', + 'CREATE_DIRS': 'false', 'STATE_DIR': '.', 'URL_PREFIX': '', 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', @@ -101,18 +101,39 @@ async def delete(request): async def connect(sid, environ): await sio.emit('all', serializer.encode(dqueue.get()), to=sid) await sio.emit('configuration', serializer.encode(config), to=sid) - if config.CUSTOM_DIR: - await sio.emit('custom_directories', serializer.encode(get_custom_directories()), to=sid) + if config.CUSTOM_DIRS: + await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid) -def get_custom_directories(): - path = pathlib.Path(config.DOWNLOAD_DIR) - # Recursively lists all subdirectories, and converts PosixPath objects to string - dirs = list(map(str, path.glob('**'))) +def get_custom_dirs(): + def recursive_dirs(base): + path = pathlib.Path(base) + + # Converts PosixPath object to string, and remove base/ prefix + def convert(p): + s = str(p) + if s.startswith(base): + s = s[len(base):] + + if s.startswith('/'): + s = s[1:] + + return s + + # Recursively lists all subdirectories of DOWNLOAD_DIR + dirs = list(filter(None, map(convert, path.glob('**')))) + + return dirs - if '.' in dirs: - dirs.remove('.') + download_dir = recursive_dirs(config.DOWNLOAD_DIR) - return {"directories": dirs} + audio_download_dir = download_dir + if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR: + audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR) + + return { + "download_dir": download_dir, + "audio_download_dir": audio_download_dir + } @routes.get(config.URL_PREFIX) def index(request): diff --git a/app/ytdl.py b/app/ytdl.py index 29fa4bd..4329147 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -228,16 +228,17 @@ class DownloadQueue: elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry: if not self.queue.exists(entry['id']): dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder) + # Keep consistent with frontend base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR if folder: - if self.config.CUSTOM_DIR != 'true': - return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIR is not true in the configuration.'} + if self.config.CUSTOM_DIRS != 'true': + return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'} dldirectory = os.path.realpath(os.path.join(base_directory, folder)) if not dldirectory.startswith(base_directory): return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{base_directory}"'} if not os.path.isdir(dldirectory): - if self.config.AUTO_CREATE_CUSTOM_DIR != 'true': - return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist, and AUTO_CREATE_CUSTOM_DIR is not true in the configuration.'} + if self.config.CREATE_DIRS != 'true': + return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{base_directory}", and CREATE_DIRS is not true in the configuration.'} os.makedirs(dldirectory, exist_ok=True) else: dldirectory = base_directory -- cgit From 63baa1fc25a7ee02832b043bb38470fe611cfb01 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Tue, 30 Aug 2022 01:22:24 -0400 Subject: Link to audio files and those with custom folders properly --- app/main.py | 1 + app/ytdl.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index de13f1e..d70d360 100644 --- a/app/main.py +++ b/app/main.py @@ -150,6 +150,7 @@ if config.URL_PREFIX != '/': routes.static(config.URL_PREFIX + 'favicon/', 'favicon') routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR) +routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR) routes.static(config.URL_PREFIX, 'ui/dist/metube') try: app.add_routes(routes) diff --git a/app/ytdl.py b/app/ytdl.py index 4329147..b7a0f41 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -208,7 +208,7 @@ class DownloadQueue: **self.config.YTDL_OPTIONS, }).extract_info(url, download=False) - async def __add_entry(self, entry, quality, format, already, folder=None): + async def __add_entry(self, entry, quality, format, folder, already): etype = entry.get('_type') or 'video' if etype == 'playlist': entries = entry['entries'] @@ -221,7 +221,7 @@ class DownloadQueue: for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, already, folder=folder)) + results.append(await self.__add_entry(etr, quality, format, folder, already)) if any(res['status'] == 'error' for res in results): return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} @@ -252,10 +252,10 @@ class DownloadQueue: await self.notifier.added(dl) return {'status': 'ok'} elif etype.startswith('url'): - return await self.add(entry['url'], quality, format, already, folder=folder) + return await self.add(entry['url'], quality, format, folder, already) return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - async def add(self, url, quality, format, already=None, folder=None): + async def add(self, url, quality, format, folder, already=None): log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}') already = set() if already is None else already if url in already: @@ -267,7 +267,7 @@ class DownloadQueue: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, already, folder=folder) + return await self.__add_entry(entry, quality, format, folder, already) async def cancel(self, ids): for id in ids: -- cgit From 202813b9edf26ee269ef808f86f1bc646782d7a9 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 19 Sep 2022 15:00:26 -0400 Subject: CREATE_DIRS -> CREATE_CUSTOM_DIRS --- app/main.py | 4 ++-- app/ytdl.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'app') diff --git a/app/main.py b/app/main.py index d70d360..27fc2b7 100644 --- a/app/main.py +++ b/app/main.py @@ -18,12 +18,12 @@ class Config: 'DOWNLOAD_DIR': '.', 'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR', 'CUSTOM_DIRS': 'true', - 'CREATE_DIRS': 'false', + 'CREATE_CUSTOM_DIRS': 'true', 'STATE_DIR': '.', 'URL_PREFIX': '', 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', - 'YTDL_OPTIONS': '{}' + 'YTDL_OPTIONS': '{}', } def __init__(self): diff --git a/app/ytdl.py b/app/ytdl.py index b7a0f41..fd1c27f 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -237,8 +237,8 @@ class DownloadQueue: if not dldirectory.startswith(base_directory): return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{base_directory}"'} if not os.path.isdir(dldirectory): - if self.config.CREATE_DIRS != 'true': - return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{base_directory}", and CREATE_DIRS is not true in the configuration.'} + if self.config.CREATE_CUSTOM_DIRS != 'true': + return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} os.makedirs(dldirectory, exist_ok=True) else: dldirectory = base_directory -- cgit From a07e1ed06c65bb473219ba4bf0372aabd35afee4 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Mon, 19 Sep 2022 15:31:46 -0400 Subject: bugfix: resolve full base directory before startswith check --- app/ytdl.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'app') diff --git a/app/ytdl.py b/app/ytdl.py index fd1c27f..97fa0a8 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -234,11 +234,12 @@ class DownloadQueue: if self.config.CUSTOM_DIRS != 'true': return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'} dldirectory = os.path.realpath(os.path.join(base_directory, folder)) - if not dldirectory.startswith(base_directory): - return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{base_directory}"'} + real_base_directory = os.path.realpath(base_directory) + if not dldirectory.startswith(real_base_directory): + return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'} if not os.path.isdir(dldirectory): if self.config.CREATE_CUSTOM_DIRS != 'true': - return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} + return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'} os.makedirs(dldirectory, exist_ok=True) else: dldirectory = base_directory -- cgit