From b66073dcd22ca22db982f8c09f0c2bf1ae5fa41e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 31 Jan 2025 09:05:11 -0500 Subject: [PATCH 1/4] Add formulas to settings --- tubesync/tubesync/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 3c350ab3..edbcf127 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -188,6 +188,12 @@ except ImportError as e: sys.stderr.write(f'Unable to import local_settings: {e}\n') sys.exit(1) +# Formulas: Do not touch! Modify the inputs instead. +DOWNLOAD_MEDIA_DELAY = 60 + int(MAX_RUN_TIME / 20) + +if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: + BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS + from .dbutils import patch_ensure_connection patch_ensure_connection() From 86c1a5fcde531e72efbd10c1d8390588c253186e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 31 Jan 2025 09:09:06 -0500 Subject: [PATCH 2/4] Remove duplicated settings and logic --- tubesync/tubesync/local_settings.py.container | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 52a51804..a5c34cdc 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -53,10 +53,7 @@ else: DEFAULT_THREADS = 1 -MAX_BACKGROUND_TASK_ASYNC_THREADS = 8 BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS)) -if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: - BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS MEDIA_ROOT = CONFIG_BASE_DIR / 'media' From d1a76ac0d1a270136b846c44d016b7a3ae88fa9b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 31 Jan 2025 09:14:09 -0500 Subject: [PATCH 3/4] Adjust download delays It turns out I was being wildly optimistic with my first guess at useful delays. --- tubesync/sync/youtube.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 665ec67a..3ea3c333 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -245,9 +245,9 @@ def download_media( 'writethumbnail': True, 'check_formats': False, 'overwrites': None, - 'sleep_interval': 30, - 'max_sleep_interval': 600, - 'sleep_interval_requests': 30, + 'sleep_interval': 10 + int(settings.DOWNLOAD_MEDIA_DELAY / 20), + 'max_sleep_interval': settings.DOWNLOAD_MEDIA_DELAY, + 'sleep_interval_requests': 5, 'paths': opts.get('paths', dict()), 'postprocessor_args': opts.get('postprocessor_args', dict()), 'postprocessor_hooks': opts.get('postprocessor_hooks', list()), From da8aa06f279a453b9107d3e936116991f50d7554 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 1 Feb 2025 11:13:44 -0500 Subject: [PATCH 4/4] Ensure `MAX_RUN_TIME` is sane Try the user's value, but set it appropriately if that fails. Background tasks will have a minimum of 10 minutes to finish. Downloading tasks will, at a minimum, sleep up to 90 seconds between downloads. Increasing the `MAX_RUN_TIME` will also increase the maximum amount of time to sleep between downloads. --- tubesync/tubesync/settings.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index edbcf127..20b53fcc 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -181,6 +181,13 @@ RENAME_ALL_SOURCES = False RENAME_SOURCES = None +# WARNING WARNING WARNING +# Below this line, the logic and formulas must remain as they are. +# Changing them is very likely to break the software in weird ways. +# To change anything, you should adjust a variable above or in the +# 'local_settings.py' file instead. +# You have been warned! + try: from .local_settings import * except ImportError as e: @@ -188,8 +195,19 @@ except ImportError as e: sys.stderr.write(f'Unable to import local_settings: {e}\n') sys.exit(1) -# Formulas: Do not touch! Modify the inputs instead. -DOWNLOAD_MEDIA_DELAY = 60 + int(MAX_RUN_TIME / 20) + +try: + MAX_RUN_TIME = int(str(MAX_RUN_TIME), base=10) +except: + # fall back to the default value from: + # https://github.com/django-background-tasks/django-background-tasks/blob/12c8f328e4ba704bd7d91534bb6569e70197b949/background_task/settings.py#L28 + MAX_RUN_TIME = 3600 + +# Tasks scheduled with `background_task` need a chance to finish +if MAX_RUN_TIME < 600: + MAX_RUN_TIME = 600 + +DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 20) if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS