Merge branch 'master' into master

3 files changed, 94 insertions, 14 deletions

diff --git a/app/dl_formats.py b/app/dl_formats.py
index c06b93b..5936b35 100644
--- a/app/dl_formats.py
+++ b/app/dl_formats.py
@@ -19,6 +19,10 @@ def get_format(format: str, quality: str) -> str:
     if format.startswith("custom:"):
         return format[7:]
 
+    if format == "thumbnail":
+        # Quality is irrelevant in this case since we skip the download
+        return "bestaudio/best"
+
     if format == "mp3":
         # Audio quality needs to be set post-download, set in opts
         return "bestaudio/best"
@@ -63,7 +67,13 @@ def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
             "preferredquality": 0 if quality == "best" else quality,
         })
         opts["writethumbnail"] = True
+        opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
         opts["postprocessors"].append({"key": "FFmpegMetadata"})
         opts["postprocessors"].append({"key": "EmbedThumbnail"})
     
+    if format == "thumbnail":
+        opts["skip_download"] = True
+        opts["writethumbnail"] = True
+        opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
+    
     return opts
diff --git a/app/main.py b/app/main.py
index 4602ef9..6fc46c6 100644
--- a/app/main.py
+++ b/app/main.py
@@ -7,6 +7,7 @@ from aiohttp import web
 import socketio

 import logging

 import json

+import pathlib

 

 from ytdl import DownloadQueueNotifier, DownloadQueue

 

@@ -16,6 +17,8 @@ class Config:
     _DEFAULTS = {

         'DOWNLOAD_DIR': '.',

         'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',

+        'CUSTOM_DIRS': 'true',

+        'CREATE_CUSTOM_DIRS': 'true',

         'STATE_DIR': '.',

         'URL_PREFIX': '',

         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',

@@ -26,12 +29,19 @@ class Config:
         'BASE_DIR': ''

     }

 

+    _BOOLEAN = ('CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS')

+

     def __init__(self):

         for k, v in self._DEFAULTS.items():

             setattr(self, k, os.environ[k] if k in os.environ else v)

         for k, v in self.__dict__.items():

             if v.startswith('%%'):

                 setattr(self, k, getattr(self, v[2:]))

+            if k in self._BOOLEAN:

+                if v not in ('true', 'false', 'True', 'False', 'on', 'off', '1', '0'):

+                    log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"')

+                    sys.exit(1)

+                setattr(self, k, v in ('true', 'True', 'on', '1'))

         if not self.URL_PREFIX.endswith('/'):

             self.URL_PREFIX += '/'

         try:

@@ -83,7 +93,8 @@ async def add(request):
     if not url or not quality:

         raise web.HTTPBadRequest()

     format = post.get('format')

-    status = await dqueue.add(url, quality, format)

+    folder = post.get('folder')

+    status = await dqueue.add(url, quality, format, folder)

     return web.Response(text=serializer.encode(status))

 

 @routes.post(config.URL_PREFIX + 'delete')

@@ -99,6 +110,40 @@ async def delete(request):
 @sio.event

 async def connect(sid, environ):

     await sio.emit('all', serializer.encode(dqueue.get()), to=sid)

+    await sio.emit('configuration', serializer.encode(config), to=sid)

+    if config.CUSTOM_DIRS:

+        await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)

+

+def get_custom_dirs():

+    def recursive_dirs(base):

+        path = pathlib.Path(base)

+

+        # Converts PosixPath object to string, and remove base/ prefix

+        def convert(p):

+            s = str(p)

+            if s.startswith(base):

+                s = s[len(base):]

+

+            if s.startswith('/'):

+                s = s[1:]

+

+            return s

+

+        # Recursively lists all subdirectories of DOWNLOAD_DIR

+        dirs = list(filter(None, map(convert, path.glob('**'))))

+

+        return dirs

+    

+    download_dir = recursive_dirs(config.DOWNLOAD_DIR)

+

+    audio_download_dir = download_dir

+    if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:

+        audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)

+    

+    return {

+        "download_dir": download_dir,

+        "audio_download_dir": audio_download_dir

+    }

 

 @routes.get(config.URL_PREFIX)

 def index(request):

@@ -115,9 +160,14 @@ if config.URL_PREFIX != '/':
 

 routes.static(config.URL_PREFIX + 'favicon/', os.path.join(config.BASE_DIR, 'favicon'))

 routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR)

+routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR)

 routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube'))

-app.add_routes(routes)

-

+try:

+    app.add_routes(routes)

+except ValueError as e:

+    if 'ui/dist/metube' in str(e):

+        raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e

+    raise e

 

 # https://github.com/aio-libs/aiohttp/pull/4615 waiting for release

 # @routes.options(config.URL_PREFIX + 'add')

diff --git a/app/ytdl.py b/app/ytdl.py
index 21b82da..0a74b14 100644
--- a/app/ytdl.py
+++ b/app/ytdl.py
@@ -6,6 +6,7 @@ import time
 import asyncio

 import multiprocessing

 import logging

+import re

 from dl_formats import get_format, get_opts

 

 log = logging.getLogger('ytdl')

@@ -27,10 +28,11 @@ class DownloadQueueNotifier:
         raise NotImplementedError

 

 class DownloadInfo:

-    def __init__(self, id, title, url, quality, format):

+    def __init__(self, id, title, url, quality, format, folder):

         self.id, self.title, self.url = id, title, url

         self.quality = quality

         self.format = format

+        self.folder = folder

         self.status = self.msg = self.percent = self.speed = self.eta = None

         self.filename = None

         self.timestamp = time.time_ns()

@@ -126,6 +128,10 @@ class Download:
             self.tmpfilename = status.get('tmpfilename')

             if 'filename' in status:

                 self.info.filename = os.path.relpath(status.get('filename'), self.download_dir)

+

+                # Set correct file extension for thumbnails

+                if(self.info.format == 'thumbnail'):

+                    self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)

             self.info.status = status['status']

             self.info.msg = status.get('msg')

             if 'downloaded_bytes' in status:

@@ -164,7 +170,7 @@ class PersistentQueue:
             return sorted(shelf.items(), key=lambda item: item[1].timestamp)

 

     def put(self, value):

-        key = value.info.id

+        key = value.info.url

         self.dict[key] = value

         with shelve.open(self.path, 'w') as shelf:

             shelf[key] = value.info

@@ -192,7 +198,7 @@ class DownloadQueue:
     

     async def __import_queue(self):

         for k, v in self.queue.saved_items():

-            await self.add(v.url, v.quality, v.format)

+            await self.add(v.url, v.quality, v.format, folder=v.folder)

 

     async def initialize(self):

         self.event = asyncio.Event()

@@ -207,7 +213,7 @@ class DownloadQueue:
             **self.config.YTDL_OPTIONS,

         }).extract_info(url, download=False)

 

-    async def __add_entry(self, entry, quality, format, already):

+    async def __add_entry(self, entry, quality, format, folder, already):

         etype = entry.get('_type') or 'video'

         if etype == 'playlist':

             entries = entry['entries']

@@ -220,14 +226,28 @@ class DownloadQueue:
                 for property in ("id", "title", "uploader", "uploader_id"):

                     if property in entry:

                         etr[f"playlist_{property}"] = entry[property]

-                results.append(await self.__add_entry(etr, quality, format, already))

+                results.append(await self.__add_entry(etr, quality, format, folder, already))

             if any(res['status'] == 'error' for res in results):

                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}

             return {'status': 'ok'}

         elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:

             if not self.queue.exists(entry['id']):

-                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format)

-                dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR

+                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder)

+                # Keep consistent with frontend

+                base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR

+                if folder:

+                    if not self.config.CUSTOM_DIRS:

+                        return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}

+                    dldirectory = os.path.realpath(os.path.join(base_directory, folder))

+                    real_base_directory = os.path.realpath(base_directory)

+                    if not dldirectory.startswith(real_base_directory):

+                        return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}

+                    if not os.path.isdir(dldirectory):

+                        if not self.config.CREATE_CUSTOM_DIRS:

+                            return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'}

+                        os.makedirs(dldirectory, exist_ok=True)

+                else:

+                    dldirectory = base_directory

                 output = self.config.OUTPUT_TEMPLATE

                 output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER

                 for property, value in entry.items():

@@ -238,11 +258,11 @@ class DownloadQueue:
                 await self.notifier.added(dl)

             return {'status': 'ok'}

         elif etype.startswith('url'):

-            return await self.add(entry['url'], quality, format, already)

+            return await self.add(entry['url'], quality, format, folder, already)

         return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}

 

-    async def add(self, url, quality, format, already=None):

-        log.info(f'adding {url}')

+    async def add(self, url, quality, format, folder, already=None):

+        log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}')

         already = set() if already is None else already

         if url in already:

             log.info('recursion detected, skipping')

@@ -253,7 +273,7 @@ class DownloadQueue:
             entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)

         except yt_dlp.utils.YoutubeDLError as exc:

             return {'status': 'error', 'msg': str(exc)}

-        return await self.__add_entry(entry, quality, format, already)

+        return await self.__add_entry(entry, quality, format, folder, already)

 

     async def cancel(self, ids):

         for id in ids: