From dfc63a2b94ff8dafbbecccb9d12d7353d571ac16 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 18:06:15 -0500 Subject: [PATCH 1/6] Reworked download progress hook Be more resilient about missing keys. Log the downloading state the first time through. --- tubesync/sync/hooks.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index cd0f208f..f14226cf 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -30,7 +30,6 @@ class PPHookStatus: def yt_dlp_progress_hook(event): hook = progress_hook.get('status', None) - filename = os.path.basename(event['filename']) if hook is None: log.error('yt_dlp_progress_hook: failed to get hook status object') return None @@ -39,31 +38,29 @@ def yt_dlp_progress_hook(event): log.warn(f'[youtube-dl] unknown event: {str(event)}') return None - if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: - return None - - if event['status'] == 'error': + filename = os.path.basename(event.get('filename', '???')) + if 'error' == event['status']: log.error(f'[youtube-dl] error occured downloading: {filename}') - elif event['status'] == 'downloading': - downloaded_bytes = event.get('downloaded_bytes', 0) - total_bytes = event.get('total_bytes', 0) + elif 'downloading' == event['status']: + downloaded_bytes = event.get('downloaded_bytes', 0) or 0 + total_bytes = event.get('total_bytes', 0) or 0 eta = event.get('_eta_str', '?').strip() percent_done = event.get('_percent_str', '?').strip() speed = event.get('_speed_str', '?').strip() total = event.get('_total_bytes_str', '?').strip() if downloaded_bytes > 0 and total_bytes > 0: - p = round((event['downloaded_bytes'] / event['total_bytes']) * 100) - if (p % 5 == 0) and p > hook.download_progress: + p = round(100 * downloaded_bytes / total_bytes) + if (0 == p % 5) and p > hook.download_progress: hook.download_progress = p log.info(f'[youtube-dl] downloading: {filename} - {percent_done} ' f'of {total} at {speed}, {eta} remaining') else: # No progress to monitor, just spam every 10 download messages instead - hook.download_progress += 1 - if hook.download_progress % 10 == 0: + if 0 == hook.download_progress % 10: log.info(f'[youtube-dl] downloading: {filename} - {percent_done} ' f'of {total} at {speed}, {eta} remaining') - elif event['status'] == 'finished': + hook.download_progress += 1 + elif 'finished' == event['status']: total_size_str = event.get('_total_bytes_str', '?').strip() elapsed_str = event.get('_elapsed_str', '?').strip() log.info(f'[youtube-dl] finished downloading: {filename} - ' From 6d71c698783283ddc6f6af09bbadb3e7bb1b2c43 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 20:02:14 -0500 Subject: [PATCH 2/6] Reset `hook.download_progress` so that it isn't above 100% --- tubesync/sync/hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index f14226cf..3ea32f9b 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -43,7 +43,8 @@ def yt_dlp_progress_hook(event): log.error(f'[youtube-dl] error occured downloading: {filename}') elif 'downloading' == event['status']: downloaded_bytes = event.get('downloaded_bytes', 0) or 0 - total_bytes = event.get('total_bytes', 0) or 0 + total_bytes_estimate = event.get('total_bytes_estimate', 0) or 0 + total_bytes = event.get('total_bytes', 0) or total_bytes_estimate eta = event.get('_eta_str', '?').strip() percent_done = event.get('_percent_str', '?').strip() speed = event.get('_speed_str', '?').strip() @@ -57,6 +58,7 @@ def yt_dlp_progress_hook(event): else: # No progress to monitor, just spam every 10 download messages instead if 0 == hook.download_progress % 10: + hook.download_progress = 0 log.info(f'[youtube-dl] downloading: {filename} - {percent_done} ' f'of {total} at {speed}, {eta} remaining') hook.download_progress += 1 From 36a9902d26fc07eae565a14d59bb0210e8363cab Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 20:31:32 -0500 Subject: [PATCH 3/6] Reset `hook.download_progress` before downloading --- tubesync/sync/youtube.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 3ea3c333..b9763856 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -274,6 +274,9 @@ def download_media( progress_hook_func = progress_hook.get('function', None) if progress_hook_func: + hook = progress_hook.get('status', None) + if hook: + hook.download_progress = 0 ytopts['progress_hooks'].append(progress_hook_func) codec_options = list() From c24178f7d67cb0f7d1c1b6bb92e53c934d5358c8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 22:24:43 -0500 Subject: [PATCH 4/6] Support multiple independent statuses --- tubesync/sync/hooks.py | 106 ++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 44 deletions(-) diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index 3ea32f9b..3244a8f5 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -29,58 +29,72 @@ class PPHookStatus: def yt_dlp_progress_hook(event): - hook = progress_hook.get('status', None) - if hook is None: - log.error('yt_dlp_progress_hook: failed to get hook status object') - return None - if event['status'] not in ProgressHookStatus.valid: log.warn(f'[youtube-dl] unknown event: {str(event)}') return None - filename = os.path.basename(event.get('filename', '???')) - if 'error' == event['status']: - log.error(f'[youtube-dl] error occured downloading: {filename}') - elif 'downloading' == event['status']: - downloaded_bytes = event.get('downloaded_bytes', 0) or 0 - total_bytes_estimate = event.get('total_bytes_estimate', 0) or 0 - total_bytes = event.get('total_bytes', 0) or total_bytes_estimate - eta = event.get('_eta_str', '?').strip() - percent_done = event.get('_percent_str', '?').strip() - speed = event.get('_speed_str', '?').strip() - total = event.get('_total_bytes_str', '?').strip() - if downloaded_bytes > 0 and total_bytes > 0: - p = round(100 * downloaded_bytes / total_bytes) - if (0 == p % 5) and p > hook.download_progress: - hook.download_progress = p - log.info(f'[youtube-dl] downloading: {filename} - {percent_done} ' - f'of {total} at {speed}, {eta} remaining') - else: - # No progress to monitor, just spam every 10 download messages instead - if 0 == hook.download_progress % 10: - hook.download_progress = 0 - log.info(f'[youtube-dl] downloading: {filename} - {percent_done} ' - f'of {total} at {speed}, {eta} remaining') - hook.download_progress += 1 - elif 'finished' == event['status']: - total_size_str = event.get('_total_bytes_str', '?').strip() - elapsed_str = event.get('_elapsed_str', '?').strip() - log.info(f'[youtube-dl] finished downloading: {filename} - ' - f'{total_size_str} in {elapsed_str}') - -def yt_dlp_postprocessor_hook(event): - if event['status'] not in PPHookStatus.valid: - log.warn(f'[youtube-dl] unknown event: {str(event)}') - return None - - postprocessor_hook['status'] = PPHookStatus(*event) - name = key = 'Unknown' if 'display_id' in event['info_dict']: key = event['info_dict']['display_id'] elif 'id' in event['info_dict']: key = event['info_dict']['id'] + filename = os.path.basename(event.get('filename', '???')) + if 'error' == event['status']: + log.error(f'[youtube-dl] error occured downloading: {filename}') + elif 'downloading' == event['status']: + # get or create the status for key + status = progress_hook['status'].get(key, None) + if status is None: + progress_hook['status'].update({key: ProgressHookStatus()}) + + downloaded_bytes = event.get('downloaded_bytes', 0) or 0 + total_bytes_estimate = event.get('total_bytes_estimate', 0) or 0 + total_bytes = event.get('total_bytes', 0) or total_bytes_estimate + eta = event.get('_eta_str', '?').strip() + percent_str = event.get('_percent_str', '?').strip() + speed = event.get('_speed_str', '?').strip() + total = event.get('_total_bytes_str', '?').strip() + percent = None + try: + percent = int(float(percent_str.rstrip('%'))) + except: + pass + if downloaded_bytes > 0 and total_bytes > 0: + percent = round(100 * downloaded_bytes / total_bytes) + if percent and (0 < percent) and (0 == percent % 5): + log.info(f'[youtube-dl] downloading: {filename} - {percent_str} ' + f'of {total} at {speed}, {eta} remaining') + status.download_progress = percent or 0 + elif 'finished' == event['status']: + # update the status for key to the finished value + status = progress_hook['status'].get(key, None) + if status is None: + progress_hook['status'].update({key: ProgressHookStatus()}) + status.download_progress = 100 + + total_size_str = event.get('_total_bytes_str', '?').strip() + elapsed_str = event.get('_elapsed_str', '?').strip() + log.info(f'[youtube-dl] finished downloading: {filename} - ' + f'{total_size_str} in {elapsed_str}') + + # clean up the status for key + if key in progress_hook['status']: + del progress_hook['status'][key] + +def yt_dlp_postprocessor_hook(event): + if event['status'] not in PPHookStatus.valid: + log.warn(f'[youtube-dl] unknown event: {str(event)}') + return None + + name = key = 'Unknown' + if 'display_id' in event['info_dict']: + key = event['info_dict']['display_id'] + elif 'id' in event['info_dict']: + key = event['info_dict']['id'] + + postprocessor_hook['status'].update({key: PPHookStatus(*event)}) + title = None if 'fulltitle' in event['info_dict']: title = event['info_dict']['fulltitle'] @@ -98,15 +112,19 @@ def yt_dlp_postprocessor_hook(event): log.debug(repr(event['info_dict'])) log.info(f'[{event["postprocessor"]}] {event["status"]} for: {name}') + if 'finished' == event['status'] and key in postprocessor_hook['status']: + del postprocessor_hook['status'][key] progress_hook = { - 'status': ProgressHookStatus(), + 'class': ProgressHookStatus(), 'function': yt_dlp_progress_hook, + 'status': dict(), } postprocessor_hook = { - 'status': PPHookStatus(), + 'class': PPHookStatus(), 'function': yt_dlp_postprocessor_hook, + 'status': dict(), } From 3ec1d2dc29dbc901c3c4c398c27ec8240195e4a4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 22:26:23 -0500 Subject: [PATCH 5/6] Initialization is done by the class now --- tubesync/sync/youtube.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index b9763856..3ea3c333 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -274,9 +274,6 @@ def download_media( progress_hook_func = progress_hook.get('function', None) if progress_hook_func: - hook = progress_hook.get('status', None) - if hook: - hook.download_progress = 0 ytopts['progress_hooks'].append(progress_hook_func) codec_options = list() From 194e84b074eef94a1640986f4fcbb47822019da9 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 6 Feb 2025 22:36:56 -0500 Subject: [PATCH 6/6] Differentiate the unknown event logs --- tubesync/sync/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index 3244a8f5..f39f7e76 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -30,7 +30,7 @@ class PPHookStatus: def yt_dlp_progress_hook(event): if event['status'] not in ProgressHookStatus.valid: - log.warn(f'[youtube-dl] unknown event: {str(event)}') + log.warn(f'[youtube-dl] unknown progress event: {str(event)}') return None name = key = 'Unknown' @@ -84,7 +84,7 @@ def yt_dlp_progress_hook(event): def yt_dlp_postprocessor_hook(event): if event['status'] not in PPHookStatus.valid: - log.warn(f'[youtube-dl] unknown event: {str(event)}') + log.warn(f'[youtube-dl] unknown postprocessor event: {str(event)}') return None name = key = 'Unknown'