Merge pull request #663 from tcely/patch-5

Reorganize hooks

tubesync/sync/hooks.py

@@ -0,0 +1,113 @@
+import os
+import yt_dlp
+
+from common.logger import log
+from django.conf import settings
+
+
+class ProgressHookStatus:
+    valid = frozenset((
+        'downloading',
+        'finished',
+        'error',
+    ))
+
+    def __init__(self):
+        self.download_progress = 0
+
+class PPHookStatus:
+    valid = frozenset((
+        'started',
+        'processing',
+        'finished',
+    ))
+
+    def __init__(self, *args, status=None, postprocessor=None, info_dict={}, **kwargs):
+        self.info = info_dict
+        self.name = postprocessor
+        self.status = status
+
+
+def yt_dlp_progress_hook(event):
+    hook = progress_hook.get('status', None)
+    filename = os.path.basename(event['filename'])
+    if hook is None:
+        log.error('yt_dlp_progress_hook: failed to get hook status object')
+        return None
+
+    if event['status'] not in ProgressHookStatus.valid:
+        log.warn(f'[youtube-dl] unknown event: {str(event)}')
+        return None
+
+    if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
+        return None
+
+    if event['status'] == 'error':
+        log.error(f'[youtube-dl] error occured downloading: {filename}')
+    elif event['status'] == 'downloading':
+        downloaded_bytes = event.get('downloaded_bytes', 0)
+        total_bytes = event.get('total_bytes', 0)
+        eta = event.get('_eta_str', '?').strip()
+        percent_done = event.get('_percent_str', '?').strip()
+        speed = event.get('_speed_str', '?').strip()
+        total = event.get('_total_bytes_str', '?').strip()
+        if downloaded_bytes > 0 and total_bytes > 0:
+            p = round((event['downloaded_bytes'] / event['total_bytes']) * 100)
+            if (p % 5 == 0) and p > hook.download_progress:
+                hook.download_progress = p
+                log.info(f'[youtube-dl] downloading: {filename} - {percent_done} '
+                         f'of {total} at {speed}, {eta} remaining')
+        else:
+            # No progress to monitor, just spam every 10 download messages instead
+            hook.download_progress += 1
+            if hook.download_progress % 10 == 0:
+                log.info(f'[youtube-dl] downloading: {filename} - {percent_done} '
+                         f'of {total} at {speed}, {eta} remaining')
+    elif event['status'] == 'finished':
+        total_size_str = event.get('_total_bytes_str', '?').strip()
+        elapsed_str = event.get('_elapsed_str', '?').strip()
+        log.info(f'[youtube-dl] finished downloading: {filename} - '
+                 f'{total_size_str} in {elapsed_str}')
+
+def yt_dlp_postprocessor_hook(event):
+    if event['status'] not in PPHookStatus.valid:
+        log.warn(f'[youtube-dl] unknown event: {str(event)}')
+        return None
+
+    postprocessor_hook['status'] = PPHookStatus(*event)
+
+    name = key = 'Unknown'
+    if 'display_id' in event['info_dict']:
+        key = event['info_dict']['display_id']
+    elif 'id' in event['info_dict']:
+        key = event['info_dict']['id']
+
+    title = None
+    if 'fulltitle' in event['info_dict']:
+        title = event['info_dict']['fulltitle']
+    elif 'title' in event['info_dict']:
+        title = event['info_dict']['title']
+
+    if title:
+        name = f'{key}: {title}'
+
+    if 'started' == event['status']:
+        if 'formats' in event['info_dict']:
+            del event['info_dict']['formats']
+        if 'automatic_captions' in event['info_dict']:
+            del event['info_dict']['automatic_captions']
+        log.debug(repr(event['info_dict']))
+
+    log.info(f'[{event["postprocessor"]}] {event["status"]} for: {name}')
+
+
+progress_hook = {
+    'status': ProgressHookStatus(),
+    'function': yt_dlp_progress_hook,
+}
+
+postprocessor_hook = {
+    'status': PPHookStatus(),
+    'function': yt_dlp_postprocessor_hook,
+}
+

tubesync/sync/youtube.py

@@ -14,8 +14,10 @@ from tempfile import TemporaryDirectory
 from urllib.parse import urlsplit, parse_qs
 
 from django.conf import settings
+from .hooks import postprocessor_hook, progress_hook
 from .utils import mkdir_p
 import yt_dlp
+from yt_dlp.utils import remove_end
 
 
 _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {})
@@ -34,7 +36,6 @@ if _youtubedl_tempdir:
     _defaults['paths'] = _paths
 
 
-
 class YouTubeError(yt_dlp.utils.DownloadError):
     '''
         Generic wrapped error for all errors that could be raised by youtube-dl.
@@ -169,54 +170,51 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file, info_json,
+# Yes, this looks odd. But, it works.
-                   sponsor_categories=None,
+# It works without also causing indentation problems.
-                   embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
+# I'll take ease of editing, thanks.
-                   write_subtitles=False, auto_subtitles=False, sub_langs='en'):
+def download_media(
+    url, media_format, extension, output_file,
+    info_json, sponsor_categories=None,
+    embed_thumbnail=False, embed_metadata=False,
+    skip_sponsors=True, write_subtitles=False,
+    auto_subtitles=False, sub_langs='en'
+):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
 
-    def hook(event):
+    opts = get_yt_opts()
-        filename = os.path.basename(event['filename'])
-
-        if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
-            return None
-
-        if event['status'] == 'error':
-            log.error(f'[youtube-dl] error occured downloading: {filename}')
-        elif event['status'] == 'downloading':
-            downloaded_bytes = event.get('downloaded_bytes', 0)
-            total_bytes = event.get('total_bytes', 0)
-            eta = event.get('_eta_str', '?').strip()
-            percent_done = event.get('_percent_str', '?').strip()
-            speed = event.get('_speed_str', '?').strip()
-            total = event.get('_total_bytes_str', '?').strip()
-            if downloaded_bytes > 0 and total_bytes > 0:
-                p = round((event['downloaded_bytes'] / event['total_bytes']) * 100)
-                if (p % 5 == 0) and p > hook.download_progress:
-                    hook.download_progress = p
-                    log.info(f'[youtube-dl] downloading: {filename} - {percent_done} '
-                             f'of {total} at {speed}, {eta} remaining')
-            else:
-                # No progress to monitor, just spam every 10 download messages instead
-                hook.download_progress += 1
-                if hook.download_progress % 10 == 0:
-                    log.info(f'[youtube-dl] downloading: {filename} - {percent_done} '
-                                f'of {total} at {speed}, {eta} remaining')
-        elif event['status'] == 'finished':
-            total_size_str = event.get('_total_bytes_str', '?').strip()
-            elapsed_str = event.get('_elapsed_str', '?').strip()
-            log.info(f'[youtube-dl] finished downloading: {filename} - '
-                     f'{total_size_str} in {elapsed_str}')
-        else:
-            log.warn(f'[youtube-dl] unknown event: {str(event)}')
-
-    hook.download_progress = 0
-
     default_opts = yt_dlp.parse_options([]).options
     pp_opts = deepcopy(default_opts)
+
+    # We fake up this option to make it easier for the user to add post processors.
+    postprocessors = opts.get('add_postprocessors', pp_opts.add_postprocessors)
+    if isinstance(postprocessors, str):
+        # NAME1[:ARGS], NAME2[:ARGS]
+        # ARGS are a semicolon ";" delimited list of NAME=VALUE
+        #
+        # This means that "," cannot be present in NAME or VALUE.
+        # If you need to support that, then use the 'postprocessors' key,
+        # in your settings dictionary instead.
+        _postprocessor_opts_parser = lambda key, val='': (
+            *(
+                item.split('=', 1) for item in (val.split(';') if val else [])
+            ),
+            ( 'key', remove_end(key, 'PP'), )
+        )
+        postprocessors = list(
+            dict(
+                _postprocessor_opts_parser( *val.split(':', 1) )
+            ) for val in map(str.strip, postprocessors.split(','))
+        )
+    if not isinstance(postprocessors, list):
+        postprocessors = list()
+    # Add any post processors configured the 'hard' way also.
+    postprocessors.extend( opts.get('postprocessors', list()) )
+
     pp_opts.__dict__.update({
+        'add_postprocessors': postprocessors,
         'embedthumbnail': embed_thumbnail,
         'addmetadata': embed_metadata,
         'addchapters': True,
@@ -227,7 +225,10 @@ def download_media(url, media_format, extension, output_file, info_json,
     })
 
     if skip_sponsors:
-        pp_opts.sponsorblock_mark.update('all,-chapter'.split(','))
+        # Let yt_dlp convert from human for us.
+        pp_opts.sponsorblock_mark = yt_dlp.parse_options(
+            ['--sponsorblock-mark=all,-chapter']
+        ).options.sponsorblock_mark
         pp_opts.sponsorblock_remove.update(sponsor_categories or {})
 
     ytopts = {
@@ -237,9 +238,7 @@ def download_media(url, media_format, extension, output_file, info_json,
         'quiet': False if settings.DEBUG else True,
         'verbose': True if settings.DEBUG else False,
         'noprogress': None if settings.DEBUG else True,
-        'progress_hooks': [hook],
         'writeinfojson': info_json,
-        'postprocessors': [],
         'writesubtitles': write_subtitles,
         'writeautomaticsub': auto_subtitles,
         'subtitleslangs': sub_langs.split(','),
@@ -249,9 +248,11 @@ def download_media(url, media_format, extension, output_file, info_json,
         'sleep_interval': 30,
         'max_sleep_interval': 600,
         'sleep_interval_requests': 30,
+        'paths': opts.get('paths', dict()),
+        'postprocessor_args': opts.get('postprocessor_args', dict()),
+        'postprocessor_hooks': opts.get('postprocessor_hooks', list()),
+        'progress_hooks': opts.get('progress_hooks', list()),
     }
-    opts = get_yt_opts()
-    ytopts['paths'] = opts.get('paths', {})
     output_dir = os.path.dirname(output_file)
     temp_dir_parent = output_dir
     temp_dir_prefix = '.yt_dlp-'
@@ -267,13 +268,20 @@ def download_media(url, media_format, extension, output_file, info_json,
         'temp': str(temp_dir_path),
     })
 
-    codec_options = []
+    postprocessor_hook_func = postprocessor_hook.get('function', None)
+    if postprocessor_hook_func:
+        ytopts['postprocessor_hooks'].append(postprocessor_hook_func)
+
+    progress_hook_func = progress_hook.get('function', None)
+    if progress_hook_func:
+        ytopts['progress_hooks'].append(progress_hook_func)
+
+    codec_options = list()
     ofn = ytopts['outtmpl']
     if 'av1-' in ofn:
         codec_options = ['-c:v', 'libsvtav1', '-preset', '8', '-crf', '35']
     elif 'vp9-' in ofn:
         codec_options = ['-c:v', 'libvpx-vp9', '-b:v', '0', '-crf', '31']
-    ytopts['postprocessor_args'] = opts.get('postprocessor_args', {})
     set_ffmpeg_codec = not (
         ytopts['postprocessor_args'] and
         ytopts['postprocessor_args']['modifychapters+ffmpeg']
@@ -283,7 +291,8 @@ def download_media(url, media_format, extension, output_file, info_json,
             'modifychapters+ffmpeg': codec_options,
         })
 
-    # create the post processors list
+    # Create the post processors list.
+    # It already included user configured post processors as well.
     ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts))
 
     opts.update(ytopts)