From a00f3e8d6579ff6f48960a45117502ff0581e558 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 5 Mar 2025 08:51:47 -0500 Subject: [PATCH 01/46] Speed up `arm64` builds This is the dependency that takes so long to compile that it dominated every other part of the build time. It's a ~12 MiB wheel when we compile from the latest source. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 851165ea..25a9fec6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -275,6 +275,7 @@ RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \ pipenv \ pkgconf \ python3 \ + python3-libsass \ python3-wheel \ curl \ less \ From d20e9956665e84393e72ce58582eaedd0867c76f Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 07:25:18 -0500 Subject: [PATCH 02/46] Create check_thumbnails.py Patch to use `check_thumbnails` instead of `check_formats` to mean test downloading every possible thumbnail URL. --- patches/yt_dlp/patches/check_thumbnails.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 patches/yt_dlp/patches/check_thumbnails.py diff --git a/patches/yt_dlp/patches/check_thumbnails.py b/patches/yt_dlp/patches/check_thumbnails.py new file mode 100644 index 00000000..5c4f8095 --- /dev/null +++ b/patches/yt_dlp/patches/check_thumbnails.py @@ -0,0 +1,42 @@ +from yt_dlp import YoutubeDL + +class PatchedYoutubeDL(YoutubeDL): + + def _sanitize_thumbnails(self, info_dict): + thumbnails = info_dict.get('thumbnails') + if thumbnails is None: + thumbnail = info_dict.get('thumbnail') + if thumbnail: + info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}] + if not thumbnails: + return + + + def check_thumbnails(thumbnails): + for t in thumbnails: + self.to_screen(f'[info] Testing thumbnail {t["id"]}') + try: + self.urlopen(HEADRequest(t['url'])) + except network_exceptions as err: + self.to_screen(f'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...') + continue + yield t + + + self._sort_thumbnails(thumbnails) + for i, t in enumerate(thumbnails): + if t.get('id') is None: + t['id'] = str(i) + if t.get('width') and t.get('height'): + t['resolution'] = '%dx%d' % (t['width'], t['height']) + t['url'] = sanitize_url(t['url']) + + + if self.params.get('check_thumbnails') is True: + info_dict['thumbnails'] = LazyList(check_thumbnails(thumbnails[::-1]), reverse=True) + else: + info_dict['thumbnails'] = thumbnails + + +YoutubeDL.__unpatched___sanitize_thumbnails = YoutubeDL._sanitize_thumbnails +YoutubeDL._sanitize_thumbnails = PatchedYoutubeDL._sanitize_thumbnails From 5f6852049692cfbea437d38b4b05773f28a2695b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 07:29:00 -0500 Subject: [PATCH 03/46] Use the new patch --- tubesync/sync/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 95eebb8a..b3d6cbbf 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -17,6 +17,7 @@ from django.conf import settings from .hooks import postprocessor_hook, progress_hook from .utils import mkdir_p import yt_dlp +import yt_dlp.patches.check_thumbnails from yt_dlp.utils import remove_end From b072b314d28bf676b08abc2b1555ceeedee39072 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 07:40:07 -0500 Subject: [PATCH 04/46] Create __init__.py --- patches/yt_dlp/patches/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 patches/yt_dlp/patches/__init__.py diff --git a/patches/yt_dlp/patches/__init__.py b/patches/yt_dlp/patches/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/patches/yt_dlp/patches/__init__.py @@ -0,0 +1 @@ + From 2e12737583e43f1881d36a1f78c51cca201c56f1 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 07:54:27 -0500 Subject: [PATCH 05/46] Copy `patches/yt_dlp` for tests --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1dd9205..ce7acbb8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,7 +28,9 @@ jobs: pip install pipenv pipenv install --system --skip-lock - name: Set up Django environment - run: cp tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py + run: | + cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py + cp -v -a -t /usr/local/lib/python3.*/dist-packages/yt_dlp/ tubesync/patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 containerise: From 1d1cb6dc1495a6f01e03b33c8145eee9512bba95 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 08:00:22 -0500 Subject: [PATCH 06/46] Use `Python3_ROOT_DIR` environment variable --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ce7acbb8..6f562686 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,7 +30,7 @@ jobs: - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py - cp -v -a -t /usr/local/lib/python3.*/dist-packages/yt_dlp/ tubesync/patches/yt_dlp/* + cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/dist-packages/yt_dlp/ tubesync/patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 containerise: From ca904f37d3e8e9095984b2498654d42d75405763 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 08:10:34 -0500 Subject: [PATCH 07/46] Show me where `yt_dlp` is loaded from --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f562686..a92016da 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,6 +30,7 @@ jobs: - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py + python -v -m yt_dlp 2>&1| grep ^Adding cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/dist-packages/yt_dlp/ tubesync/patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 From af7d6292af196e3af19440ef7f04c384be812cc5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 08:16:55 -0500 Subject: [PATCH 08/46] Find should work for old versions too --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a92016da..72be3e4e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,8 +30,8 @@ jobs: - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py - python -v -m yt_dlp 2>&1| grep ^Adding - cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/dist-packages/yt_dlp/ tubesync/patches/yt_dlp/* + find /usr /opt -name yt_dlp -type d -print + cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ tubesync/patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 containerise: From 629ff5cfc81317a20a073a847ddb3cc08a185b18 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 08:23:38 -0500 Subject: [PATCH 09/46] Use the correct source path --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 72be3e4e..6068cab1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,8 +30,7 @@ jobs: - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py - find /usr /opt -name yt_dlp -type d -print - cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ tubesync/patches/yt_dlp/* + cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 containerise: From cffe8348c3d20523cec186ccbb343dca3d91a5c4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 09:26:40 -0500 Subject: [PATCH 10/46] Passthrough module for `patch` --- patches/yt_dlp/patches/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/patches/yt_dlp/patches/__init__.py b/patches/yt_dlp/patches/__init__.py index 8b137891..f2d40a97 100644 --- a/patches/yt_dlp/patches/__init__.py +++ b/patches/yt_dlp/patches/__init__.py @@ -1 +1,5 @@ +from yt_dlp.compat.compat_utils import passthrough_module + +passthrough_module(__name__, '.patch') +del passthrough_module From a8fd6ee00beebf79573f57cedde8eaa892ad4d47 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 09:28:24 -0500 Subject: [PATCH 11/46] Adjust import --- tubesync/sync/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index b3d6cbbf..edcb3c0e 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -17,7 +17,7 @@ from django.conf import settings from .hooks import postprocessor_hook, progress_hook from .utils import mkdir_p import yt_dlp -import yt_dlp.patches.check_thumbnails +import yt_dlp.patch.check_thumbnails from yt_dlp.utils import remove_end From 19c301ad76db2bc3f18e1bb05c2d7ee01b1d762c Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 10:08:32 -0500 Subject: [PATCH 12/46] Rename patches to patch --- patches/yt_dlp/{patches => patch}/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename patches/yt_dlp/{patches => patch}/__init__.py (100%) diff --git a/patches/yt_dlp/patches/__init__.py b/patches/yt_dlp/patch/__init__.py similarity index 100% rename from patches/yt_dlp/patches/__init__.py rename to patches/yt_dlp/patch/__init__.py From 4f9e0bf949b3f52db72d614b7eacf83535eb37d0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 10:09:26 -0500 Subject: [PATCH 13/46] Rename patches to patch --- patches/yt_dlp/{patches => patch}/check_thumbnails.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename patches/yt_dlp/{patches => patch}/check_thumbnails.py (100%) diff --git a/patches/yt_dlp/patches/check_thumbnails.py b/patches/yt_dlp/patch/check_thumbnails.py similarity index 100% rename from patches/yt_dlp/patches/check_thumbnails.py rename to patches/yt_dlp/patch/check_thumbnails.py From b553443255e7788ad8e635835333b8fff0ec09cf Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 10:31:31 -0500 Subject: [PATCH 14/46] Copy patches before trying to use them --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95b909ce..99003e01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -346,6 +346,14 @@ RUN --mount=type=tmpfs,target=/cache \ COPY tubesync /app COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py +# patch background_task +COPY patches/background_task/ \ + /usr/local/lib/python3/dist-packages/background_task/ + +# patch yt_dlp +COPY patches/yt_dlp/ \ + /usr/local/lib/python3/dist-packages/yt_dlp/ + # Build app RUN set -x && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database @@ -371,14 +379,6 @@ RUN set -x && \ # Copy root COPY config/root / -# patch background_task -COPY patches/background_task/ \ - /usr/local/lib/python3/dist-packages/background_task/ - -# patch yt_dlp -COPY patches/yt_dlp/ \ - /usr/local/lib/python3/dist-packages/yt_dlp/ - # Create a healthcheck HEALTHCHECK --interval=1m --timeout=10s --start-period=3m CMD ["/app/healthcheck.py", "http://127.0.0.1:8080/healthcheck"] From 7c0891c70370ba3b58b8cb3a8d5286308144a6e0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 10:45:28 -0500 Subject: [PATCH 15/46] Link to the `python3` version immediately after installing it --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99003e01..2343d962 100644 --- a/Dockerfile +++ b/Dockerfile @@ -279,6 +279,8 @@ RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \ curl \ less \ && \ + # Link to the current python3 version + ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ @@ -369,8 +371,6 @@ RUN set -x && \ mkdir -v -p /config/cache/pycache && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ - # Link to the current python3 version - ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \ # Append software versions ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${ffmpeg_version}" && \ From c2aa9a4b9f358feadda73418c66720175d58c282 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 11:07:13 -0500 Subject: [PATCH 16/46] Explicitly turn off checking of thumbnails --- tubesync/sync/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index edcb3c0e..483142f3 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -155,6 +155,7 @@ def get_media_info(url, days=None): 'logger': log, 'extract_flat': True, 'check_formats': True, + 'check_thumbnails': False, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { 'youtube': {'formats': ['missing_pot']}, From 01a9f07d5896e40b7856969cd882d23fd0c5cac7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Mar 2025 11:59:44 -0500 Subject: [PATCH 17/46] Import missing functions from `yt_dlp.utils` --- patches/yt_dlp/patch/check_thumbnails.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/patches/yt_dlp/patch/check_thumbnails.py b/patches/yt_dlp/patch/check_thumbnails.py index 5c4f8095..25723bb6 100644 --- a/patches/yt_dlp/patch/check_thumbnails.py +++ b/patches/yt_dlp/patch/check_thumbnails.py @@ -1,4 +1,5 @@ from yt_dlp import YoutubeDL +from yt_dlp.utils import sanitize_url, LazyList class PatchedYoutubeDL(YoutubeDL): @@ -14,7 +15,7 @@ class PatchedYoutubeDL(YoutubeDL): def check_thumbnails(thumbnails): for t in thumbnails: - self.to_screen(f'[info] Testing thumbnail {t["id"]}') + self.to_screen(f'[info] Testing thumbnail {t["id"]}: {t["url"]!r}') try: self.urlopen(HEADRequest(t['url'])) except network_exceptions as err: From 3c94e5a0b33c113c02a5bf69a8c4f7427af8f92d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 06:56:39 -0400 Subject: [PATCH 18/46] Add and use `getenv` `os.getenv` makes no guarantees about the return type for default values. --- tubesync/tubesync/local_settings.py.container | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 4b73b7d7..d1021cd9 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -5,24 +5,49 @@ from urllib.parse import urljoin from common.utils import parse_database_connection_string +def getenv(key, default=None, /, *, string=True, integer=False): + ''' + Calls `os.getenv` and guarantees that a string is returned + ''' + + unsupported_type_msg = 'Unsupported type for positional argument, "{}": {}' + assert isinstance(key, (str,)), unsupported_type_msg.format('key', type(key)) + assert isinstance(default, (str, bool, float, int, None.__class__,)), unsupported_type_msg.format('default', type(default)) + + d = default + k = key + if default is not None: + d = str(default) + import os # just in case it wasn't already imported + + r = os.getenv(k, d) + if r is None: + if string: r = str() + if integer: r = int() + elif integer: + r = int(float(r)) + return r + + BASE_DIR = Path(__file__).resolve().parent.parent ROOT_DIR = Path('/') CONFIG_BASE_DIR = ROOT_DIR / 'config' DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads' -DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None) -STATIC_URL = str(os.getenv('DJANGO_STATIC_URL', '/static/')) +DJANGO_URL_PREFIX = getenv('DJANGO_URL_PREFIX', str()).strip() +STATIC_URL = getenv('DJANGO_STATIC_URL', '/static/').strip() if DJANGO_URL_PREFIX and STATIC_URL: STATIC_URL = urljoin(DJANGO_URL_PREFIX, STATIC_URL[1:]) # This is not ever meant to be a public web interface so this isn't too critical -SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', 'tubesync-django-secret')) +SECRET_KEY = getenv('DJANGO_SECRET_KEY', 'tubesync-django-secret') -ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '*')) +ALLOWED_HOSTS_STR = getenv('TUBESYNC_HOSTS', '*') ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',') -DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False -FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) +DEBUG_STR = getenv('TUBESYNC_DEBUG', False) +DEBUG = True if 'true' == DEBUG_STR.strip().lower() else False +FORCE_SCRIPT_NAME = getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) database_dict = {} @@ -34,7 +59,8 @@ if database_connection_env: if database_dict: print(f'Using database connection: {database_dict["ENGINE"]}://' f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:' - f'{database_dict["PORT"]}/{database_dict["NAME"]}', file=sys.stdout) + f'{database_dict["PORT"]}/{database_dict["NAME"]}', + file=sys.stdout, flush=True) DATABASES = { 'default': database_dict, } @@ -60,7 +86,7 @@ else: DEFAULT_THREADS = 1 -BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS)) +BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True) MEDIA_ROOT = CONFIG_BASE_DIR / 'media' From c2653b76a92081698ff23b8073a9d8e21f07ba06 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 09:34:36 -0400 Subject: [PATCH 19/46] Add `getenv` to `common.utils` --- tubesync/common/utils.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 95efd9f3..007f3f0d 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -1,3 +1,4 @@ +import os import string from datetime import datetime from urllib.parse import urlunsplit, urlencode, urlparse @@ -6,6 +7,44 @@ from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError +def getenv(key, default=None, /, *, integer=False, string=True): + ''' + Guarantees a returned type from calling `os.getenv` + The caller can request the integer type, + or use the default string type. + ''' + + args = dict(key=key, default=default, integer=integer, string=string) + supported_types = dict(zip(args.keys(), ( + (str,), # key + ( + bool, + float, + int, + str, + None.__class__, + ), # default + (bool,) * (len(args.keys()) - 2), + ))) + unsupported_type_msg = 'Unsupported type for positional argument, "{}": {}' + for k, t in supported_types.items(): + v = args[k] + assert isinstance(v, t), unsupported_type_msg.format(k, type(v)) + + d = str(default) if default is not None else None + + # just in case `os` wasn't already imported + import os + + r = os.getenv(key, d) + if r is None: + if string: r = str() + if integer: r = int() + elif integer: + r = int(float(r)) + return r + + def parse_database_connection_string(database_connection_string): ''' Parses a connection string in a URL style format, such as: From 7315cb985398dc2c4a375d44649ef698148b440f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 09:37:11 -0400 Subject: [PATCH 20/46] Remove `getenv` from local_settings.py.container --- tubesync/tubesync/local_settings.py.container | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index d1021cd9..c2986ac2 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -2,31 +2,7 @@ import os import sys from pathlib import Path from urllib.parse import urljoin -from common.utils import parse_database_connection_string - - -def getenv(key, default=None, /, *, string=True, integer=False): - ''' - Calls `os.getenv` and guarantees that a string is returned - ''' - - unsupported_type_msg = 'Unsupported type for positional argument, "{}": {}' - assert isinstance(key, (str,)), unsupported_type_msg.format('key', type(key)) - assert isinstance(default, (str, bool, float, int, None.__class__,)), unsupported_type_msg.format('default', type(default)) - - d = default - k = key - if default is not None: - d = str(default) - import os # just in case it wasn't already imported - - r = os.getenv(k, d) - if r is None: - if string: r = str() - if integer: r = int() - elif integer: - r = int(float(r)) - return r +from common.utils import getenv, parse_database_connection_string BASE_DIR = Path(__file__).resolve().parent.parent From eabcb36aaa58e2f56efb83023bba352ba04b73c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 09:43:01 -0400 Subject: [PATCH 21/46] Switch to `common.utils.getenv` in settings.py --- tubesync/tubesync/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a9f4061c..ff88a669 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,5 +1,5 @@ -import os from pathlib import Path +from common.utils import getenv BASE_DIR = Path(__file__).resolve().parent.parent @@ -97,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = os.getenv('TZ', 'UTC') +TIME_ZONE = getenv('TZ', 'UTC') USE_I18N = True USE_L10N = True USE_TZ = True From e5c0abbdca62a7dc4449d0daf93ec6ec177ca97d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 09:45:46 -0400 Subject: [PATCH 22/46] `os` was imported --- tubesync/common/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 007f3f0d..acb55561 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -33,9 +33,6 @@ def getenv(key, default=None, /, *, integer=False, string=True): d = str(default) if default is not None else None - # just in case `os` wasn't already imported - import os - r = os.getenv(key, d) if r is None: if string: r = str() From c0115c0431ab22f05f475fc8d1d495804dbc6606 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 10:38:04 -0400 Subject: [PATCH 23/46] Update local_settings.py.container --- tubesync/tubesync/local_settings.py.container | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index c2986ac2..629bb5ff 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -1,4 +1,3 @@ -import os import sys from pathlib import Path from urllib.parse import urljoin @@ -9,7 +8,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ROOT_DIR = Path('/') CONFIG_BASE_DIR = ROOT_DIR / 'config' DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads' -DJANGO_URL_PREFIX = getenv('DJANGO_URL_PREFIX', str()).strip() +DJANGO_URL_PREFIX = getenv('DJANGO_URL_PREFIX').strip() STATIC_URL = getenv('DJANGO_STATIC_URL', '/static/').strip() if DJANGO_URL_PREFIX and STATIC_URL: STATIC_URL = urljoin(DJANGO_URL_PREFIX, STATIC_URL[1:]) @@ -27,7 +26,7 @@ FORCE_SCRIPT_NAME = getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) database_dict = {} -database_connection_env = os.getenv('DATABASE_CONNECTION', '') +database_connection_env = getenv('DATABASE_CONNECTION') if database_connection_env: database_dict = parse_database_connection_string(database_connection_env) @@ -72,14 +71,14 @@ YOUTUBE_DL_TEMPDIR = DOWNLOAD_ROOT / 'cache' COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' -HEALTHCHECK_FIREWALL_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_FIREWALL', 'True')).strip().lower() -HEALTHCHECK_FIREWALL = True if HEALTHCHECK_FIREWALL_STR == 'true' else False -HEALTHCHECK_ALLOWED_IPS_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_ALLOWED_IPS', '127.0.0.1')) +HEALTHCHECK_FIREWALL_STR = getenv('TUBESYNC_HEALTHCHECK_FIREWALL', True) +HEALTHCHECK_FIREWALL = ( 'true' == HEALTHCHECK_FIREWALL_STR.strip().lower() ) +HEALTHCHECK_ALLOWED_IPS_STR = getenv('TUBESYNC_HEALTHCHECK_ALLOWED_IPS', '127.0.0.1') HEALTHCHECK_ALLOWED_IPS = HEALTHCHECK_ALLOWED_IPS_STR.split(',') -BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip() -BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip() +BASICAUTH_USERNAME = getenv('HTTP_USER').strip() +BASICAUTH_PASSWORD = getenv('HTTP_PASS').strip() if BASICAUTH_USERNAME and BASICAUTH_PASSWORD: BASICAUTH_DISABLE = False BASICAUTH_USERS = { @@ -90,25 +89,25 @@ else: BASICAUTH_USERS = {} -SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() -SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False +SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = getenv('TUBESYNC_DIRECTORY_PREFIX', True) +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = ( 'true' == SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR.strip().lower() ) -SHRINK_NEW_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_NEW', 'false').strip().lower() -SHRINK_NEW_MEDIA_METADATA = ( 'true' == SHRINK_NEW_MEDIA_METADATA_STR ) -SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip().lower() -SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR ) +SHRINK_NEW_MEDIA_METADATA_STR = getenv('TUBESYNC_SHRINK_NEW', False) +SHRINK_NEW_MEDIA_METADATA = ( 'true' == SHRINK_NEW_MEDIA_METADATA_STR.strip().lower() ) +SHRINK_OLD_MEDIA_METADATA_STR = getenv('TUBESYNC_SHRINK_OLD', False) +SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR.strip().lower() ) # TUBESYNC_RENAME_ALL_SOURCES: True or False -RENAME_ALL_SOURCES_STR = os.getenv('TUBESYNC_RENAME_ALL_SOURCES', 'False').strip().lower() -RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR ) +RENAME_ALL_SOURCES_STR = getenv('TUBESYNC_RENAME_ALL_SOURCES', False) +RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR.strip().lower() ) # TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories -RENAME_SOURCES_STR = os.getenv('TUBESYNC_RENAME_SOURCES', '') +RENAME_SOURCES_STR = getenv('TUBESYNC_RENAME_SOURCES') RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None -VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) +VIDEO_HEIGHT_CUTOFF = getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", 240, integer=True) # ensure that the current directory exists @@ -119,4 +118,11 @@ old_youtube_cache_dirs = list(YOUTUBE_DL_CACHEDIR.parent.glob('youtube-*')) old_youtube_cache_dirs.extend(list(YOUTUBE_DL_CACHEDIR.parent.glob('youtube/youtube-*'))) for cache_dir in old_youtube_cache_dirs: cache_dir.rename(YOUTUBE_DL_CACHEDIR / cache_dir.name) +# try to remove the old, hopefully empty, directory +empty_old_youtube_dir = YOUTUBE_DL_CACHEDIR.parent / 'youtube' +if empty_old_youtube_dir.is_dir(): + try: + empty_old_youtube_dir.rmdir() + except: + pass From be71f8cc10d4cc54f973590c50080ea1e58ce338 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 15:42:18 -0400 Subject: [PATCH 24/46] Display the shorter engine instead of driver --- tubesync/tubesync/local_settings.py.container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 629bb5ff..cc20f73b 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -32,7 +32,7 @@ if database_connection_env: if database_dict: - print(f'Using database connection: {database_dict["ENGINE"]}://' + print(f'Using database connection: {database_dict["DRIVER"]}://' f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:' f'{database_dict["PORT"]}/{database_dict["NAME"]}', file=sys.stdout, flush=True) From 58472f77858417dd2f408d9d2f8f2694fae83b3f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 21:47:48 -0400 Subject: [PATCH 25/46] Add explicit transactions for certain tasks --- tubesync/sync/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index fdc954a3..d85c6be7 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -17,6 +17,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.utils import timezone +from django.db.tansaction import atomic from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ from background_task import background @@ -179,6 +180,7 @@ def cleanup_removed_media(source, videos): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -221,7 +223,8 @@ def index_source_task(source_id): if published_dt is not None: media.published = published_dt try: - media.save() + with atomic(): + media.save() log.debug(f'Indexed media: {source} / {media}') # log the new media instances new_media_instance = ( @@ -611,9 +614,10 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - for media in mqs: - if media.uuid not in already_saved: - media.save() + with atomic(): + for media in mqs: + if media.uuid not in already_saved: + media.save() @background(schedule=60, remove_existing_tasks=True) @@ -626,6 +630,7 @@ def rename_media(media_id): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def rename_all_media_for_source(source_id): try: source = Source.objects.get(pk=source_id) @@ -653,7 +658,8 @@ def rename_all_media_for_source(source_id): downloaded=True, ) for media in mqs: - media.rename_files() + with atomic(): + media.rename_files() @background(schedule=60, remove_existing_tasks=True) From 84d42fb2ab1f94a17f99f4ce19b71e3251120515 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:02:44 -0400 Subject: [PATCH 26/46] Add more logging to `get_media_info` --- tubesync/sync/youtube.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 95eebb8a..a739a4f7 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -156,10 +156,13 @@ def get_media_info(url, days=None): 'check_formats': True, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { - 'youtube': {'formats': ['missing_pot']}, 'youtubetab': {'approximate_date': ['true']}, }, + 'sleep_interval_requests': 2, + 'verbose': True if settings.DEBUG else False, }) + if start: + log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}') response = {} with yt_dlp.YoutubeDL(opts) as y: try: From 4b3605f65ee46c4647a66ce7acab19258a9b6373 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:31:56 -0400 Subject: [PATCH 27/46] Use a temporary directory for testing formats --- tubesync/sync/youtube.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index a739a4f7..48cff0c9 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -146,6 +146,14 @@ def get_media_info(url, days=None): f'yesterday-{days!s}days' if days else None ) opts = get_yt_opts() + paths = opts.get('paths', dict()) + if 'temp' in paths: + temp_dir_obj = TemporaryDirectory(prefix='.yt_dlp-', dir=paths['temp']) + temp_dir_path = Path(temp_dir_obj.name) + (temp_dir_path / '.ignore').touch(exist_ok=True) + paths.update({ + 'temp': str(temp_dir_path), + }) opts.update({ 'ignoreerrors': False, # explicitly set this to catch exceptions 'ignore_no_formats_error': False, # we must fail first to try again with this enabled @@ -158,6 +166,7 @@ def get_media_info(url, days=None): 'extractor_args': { 'youtubetab': {'approximate_date': ['true']}, }, + 'paths': paths, 'sleep_interval_requests': 2, 'verbose': True if settings.DEBUG else False, }) From 79ed138aa103de6969b4220df3b6f36c1db97adf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:45:49 -0400 Subject: [PATCH 28/46] fixup: typo --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d85c6be7..4a1884d8 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -17,7 +17,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.utils import timezone -from django.db.tansaction import atomic +from django.db.transaction import atomic from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ from background_task import background From 27955d7b7b51533a77a6640c2d8c1c698b04ee7a Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 10 Mar 2025 16:42:25 -0400 Subject: [PATCH 29/46] Improve checking media efficiency --- tubesync/sync/signals.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index c03a4f72..404974c7 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -167,6 +167,7 @@ def task_task_failed(sender, task_id, completed_task, **kwargs): @receiver(post_save, sender=Media) def media_post_save(sender, instance, created, **kwargs): + media = instance # If the media is skipped manually, bail. if instance.manual_skip: return @@ -176,12 +177,27 @@ def media_post_save(sender, instance, created, **kwargs): # Reset the skip flag if the download cap has changed if the media has not # already been downloaded downloaded = instance.downloaded + existing_media_metadata_task = get_media_metadata_task(str(instance.pk)) + existing_media_download_task = get_media_download_task(str(instance.pk)) if not downloaded: - skip_changed = filter_media(instance) + # the decision to download was already made if a download task exists + if not existing_media_download_task: + # Recalculate the "can_download" flag, this may + # need to change if the source specifications have been changed + if instance.metadata: + if instance.get_format_str(): + if not instance.can_download: + instance.can_download = True + can_download_changed = True + else: + if instance.can_download: + instance.can_download = False + can_download_changed = True + # Recalculate the "skip_changed" flag + skip_changed = filter_media(instance) else: # Downloaded media might need to be renamed # Check settings before any rename tasks are scheduled - media = instance rename_sources_setting = settings.RENAME_SOURCES or list() create_rename_task = ( ( @@ -200,18 +216,6 @@ def media_post_save(sender, instance, created, **kwargs): remove_existing_tasks=True ) - # Recalculate the "can_download" flag, this may - # need to change if the source specifications have been changed - if instance.metadata: - if instance.get_format_str(): - if not instance.can_download: - instance.can_download = True - can_download_changed = True - else: - if instance.can_download: - instance.can_download = False - can_download_changed = True - existing_media_metadata_task = get_media_metadata_task(str(instance.pk)) # If the media is missing metadata schedule it to be downloaded if not (instance.skip or instance.metadata or existing_media_metadata_task): log.info(f'Scheduling task to download metadata for: {instance.url}') @@ -239,7 +243,6 @@ def media_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) - existing_media_download_task = get_media_download_task(str(instance.pk)) # If the media has not yet been downloaded schedule it to be downloaded if not (instance.media_file_exists or instance.filepath.exists() or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. From 8d65b5785216c80efd6881c5a9415fe5e91d259f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 07:26:35 -0400 Subject: [PATCH 30/46] Remove explicit `libsass` We need this to use the package added in #808. --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index b0aad1e4..9f8adf33 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,6 @@ autopep8 = "*" [packages] django = "*" django-sass-processor = "*" -libsass = "*" pillow = "*" whitenoise = "*" gunicorn = "*" From e174c42bf50bb2273298ae834d1a935bc0c15155 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 08:14:23 -0400 Subject: [PATCH 31/46] Remove explicit `django-compressor` Also, add some of the useful optional dependencies. --- Pipfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 9f8adf33..2b051bf8 100644 --- a/Pipfile +++ b/Pipfile @@ -8,11 +8,10 @@ autopep8 = "*" [packages] django = "*" -django-sass-processor = "*" +django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" gunicorn = "*" -django-compressor = "*" httptools = "*" django-background-tasks = ">=1.2.8" django-basicauth = "*" @@ -21,3 +20,5 @@ mysqlclient = "*" yt-dlp = "*" requests = {extras = ["socks"], version = "*"} emoji = "*" +brotli = "*" +html5lib = "*" From 239cfca534994692b97665845b906e18d8372860 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 08:24:02 -0400 Subject: [PATCH 32/46] Use socks support from the operating system --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 52f20479..47892191 100644 --- a/Dockerfile +++ b/Dockerfile @@ -276,6 +276,7 @@ RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \ pkgconf \ python3 \ python3-libsass \ + python3-python-socks \ python3-wheel \ curl \ less \ From 7db0048f80e8869d25764b031ff430420b91a0e3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 08:25:57 -0400 Subject: [PATCH 33/46] Remove requests[socks] from Pipfile --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index 9f8adf33..a6bc52d8 100644 --- a/Pipfile +++ b/Pipfile @@ -19,5 +19,4 @@ django-basicauth = "*" psycopg2-binary = "*" mysqlclient = "*" yt-dlp = "*" -requests = {extras = ["socks"], version = "*"} emoji = "*" From 092e5ef54cb78c28834ee39fe35dd1f99ca194da Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 08:38:25 -0400 Subject: [PATCH 34/46] Add `python3-socks` --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 47892191..9f579b58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -277,6 +277,7 @@ RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \ python3 \ python3-libsass \ python3-python-socks \ + python3-socks \ python3-wheel \ curl \ less \ From 2cd33672afa59939c59f4cebcef80a533fe40966 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 09:05:38 -0400 Subject: [PATCH 35/46] Remove `python3-python-socks` --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f579b58..43aebe28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -276,7 +276,6 @@ RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \ pkgconf \ python3 \ python3-libsass \ - python3-python-socks \ python3-socks \ python3-wheel \ curl \ From aa54a88cdb1ae9cfe35d779378e6ce2a9e9668f5 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 09:25:57 -0400 Subject: [PATCH 36/46] Add `urllib3[socks]` and `requests[socks]` These each have slightly different version requirements for `PySocks`. --- Pipfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pipfile b/Pipfile index a6bc52d8..0080822f 100644 --- a/Pipfile +++ b/Pipfile @@ -18,5 +18,7 @@ django-background-tasks = ">=1.2.8" django-basicauth = "*" psycopg2-binary = "*" mysqlclient = "*" +urllib3 = {extras = ["socks"], version = "*"} +requests = {extras = ["socks"], version = "*"} yt-dlp = "*" emoji = "*" From f0f7edfd479827e081960fe0426486b06fe9c688 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 09:28:26 -0400 Subject: [PATCH 37/46] Add `PySocks` Upgrade to the latest, if the OS version is too old. --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 0080822f..907755b7 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,7 @@ django-background-tasks = ">=1.2.8" django-basicauth = "*" psycopg2-binary = "*" mysqlclient = "*" +PySocks = "*" urllib3 = {extras = ["socks"], version = "*"} requests = {extras = ["socks"], version = "*"} yt-dlp = "*" From af0e300ef1750b2d566252d731a47e638063ebdf Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 16:59:33 -0400 Subject: [PATCH 38/46] Download metadata before indexing another source --- tubesync/sync/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4a1884d8..498d73fe 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -234,6 +234,13 @@ def index_source_task(source_id): ) if new_media_instance: log.info(f'Indexed new media: {source} / {media}') + log.info(f'Scheduling task to download metadata for: {media.url}') + verbose_name = _('Downloading metadata for "{}"') + download_media_metadata( + str(media.pk), + priority=9, + verbose_name=verbose_name.format(media.pk), + ) except IntegrityError as e: log.error(f'Index media failed: {source} / {media} with "{e}"') # Tack on a cleanup of old completed tasks From 3c714859dbcb9716ed0ed416b77668c1a1a7a568 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Mar 2025 17:14:44 -0400 Subject: [PATCH 39/46] Rename then check media --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 404974c7..6f43e1bc 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -129,7 +129,7 @@ def source_post_save(sender, instance, created, **kwargs): verbose_name = _('Checking all media for source "{}"') save_all_media_for_source( str(instance.pk), - priority=9, + priority=25, verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) @@ -211,7 +211,7 @@ def media_post_save(sender, instance, created, **kwargs): rename_media( str(media.pk), queue=str(media.pk), - priority=16, + priority=20, verbose_name=verbose_name.format(media.key, media.name), remove_existing_tasks=True ) From ef5b939caf2a3f8270e3f18823a5abdaebc02dd7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Mar 2025 13:02:55 -0400 Subject: [PATCH 40/46] Check the copied `nginx` configuration Checking before the copy doesn't help. Fixes #804 --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 43aebe28..81b3e31e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -352,8 +352,6 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings. RUN set -x && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database rm -rf /app/db.sqlite3 && \ - # Check nginx configuration - nginx -t && \ # Run any required app commands /usr/bin/python3 -B /app/manage.py compilescss && \ /usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \ @@ -373,6 +371,9 @@ RUN set -x && \ # Copy root COPY config/root / +# Check nginx configuration copied from config/root/etc +RUN set -x && nginx -t + # patch background_task COPY patches/background_task/ \ /usr/local/lib/python3/dist-packages/background_task/ From 2640a4ae7cb4b7d4d7164ef4f25876955f8e8551 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Thu, 13 Mar 2025 02:55:09 +0900 Subject: [PATCH 41/46] docs: update README.md Rasperry Pi -> Raspberry Pi --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af3cd910..17367a4a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ currently just Plex, to complete the PVR experience. TubeSync is designed to be run in a container, such as via Docker or Podman. It also works in a Docker Compose stack. `amd64` (most desktop PCs and servers) and `arm64` -(modern ARM computers, such as the Rasperry Pi 3 or later) are supported. +(modern ARM computers, such as the Raspberry Pi 3 or later) are supported. Example (with Docker on *nix): @@ -356,7 +356,7 @@ etc.). Configuration of this is beyond the scope of this README. Only two are supported, for the moment: - `amd64` (most desktop PCs and servers) - `arm64` -(modern ARM computers, such as the Rasperry Pi 3 or later) +(modern ARM computers, such as the Raspberry Pi 3 or later) Others may be made available, if there is demand. From d1cc05a8f41df3f542558ff60d085f2b2f649851 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Mar 2025 20:05:58 -0400 Subject: [PATCH 42/46] Use '/tmp' instead of '/' for odd cases --- tubesync/sync/signals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 6f43e1bc..338e912e 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -43,6 +43,8 @@ def source_pre_save(sender, instance, **kwargs): work_directory = existing_dirpath for _count in range(parents_count, 0, -1): work_directory = work_directory.parent + if Path(existing_dirpath.root).resolve(strict=True) == Path(work_directory).resolve(strict=True): + work_directory = Path('/tmp') with TemporaryDirectory(suffix=('.'+new_dirpath.name), prefix='.tmp.', dir=work_directory) as tmp_dir: tmp_dirpath = Path(tmp_dir) existed = None From f7dbd0cf8263e504f8dc64e1e8926d55be617698 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Mar 2025 20:17:18 -0400 Subject: [PATCH 43/46] We can't rename across devices so don't leave `/downloads` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 338e912e..c348c714 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -44,7 +44,7 @@ def source_pre_save(sender, instance, **kwargs): for _count in range(parents_count, 0, -1): work_directory = work_directory.parent if Path(existing_dirpath.root).resolve(strict=True) == Path(work_directory).resolve(strict=True): - work_directory = Path('/tmp') + work_directory = Path('/downloads') with TemporaryDirectory(suffix=('.'+new_dirpath.name), prefix='.tmp.', dir=work_directory) as tmp_dir: tmp_dirpath = Path(tmp_dir) existed = None From 469858a33aea2cae2137850ef86ae0c4e9b63aac Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Mar 2025 20:28:08 -0400 Subject: [PATCH 44/46] Use the `DOWNLOADS_BASE_DIR` setting --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index c348c714..555ea9be 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -44,7 +44,7 @@ def source_pre_save(sender, instance, **kwargs): for _count in range(parents_count, 0, -1): work_directory = work_directory.parent if Path(existing_dirpath.root).resolve(strict=True) == Path(work_directory).resolve(strict=True): - work_directory = Path('/downloads') + work_directory = Path(settings.DOWNLOADS_BASE_DIR) with TemporaryDirectory(suffix=('.'+new_dirpath.name), prefix='.tmp.', dir=work_directory) as tmp_dir: tmp_dirpath = Path(tmp_dir) existed = None From 291631f76fe254b3eaf20707e7806cd12652db5b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Mar 2025 20:40:14 -0400 Subject: [PATCH 45/46] Use `DOWNLOAD_ROOT` setting instead --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 555ea9be..8bea1ce2 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -43,8 +43,8 @@ def source_pre_save(sender, instance, **kwargs): work_directory = existing_dirpath for _count in range(parents_count, 0, -1): work_directory = work_directory.parent - if Path(existing_dirpath.root).resolve(strict=True) == Path(work_directory).resolve(strict=True): - work_directory = Path(settings.DOWNLOADS_BASE_DIR) + if not Path(work_directory).resolve(strict=True).is_relative_to(Path(settings.DOWNLOAD_ROOT)): + work_directory = Path(settings.DOWNLOAD_ROOT) with TemporaryDirectory(suffix=('.'+new_dirpath.name), prefix='.tmp.', dir=work_directory) as tmp_dir: tmp_dirpath = Path(tmp_dir) existed = None From 487e8011517580706c4eb155fb55cf1fe9402e0d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 13 Mar 2025 03:23:27 -0400 Subject: [PATCH 46/46] Increase episode number calculation speed --- tubesync/sync/models.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2e802599..9ab126db 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1507,17 +1507,35 @@ class Media(models.Model): def calculate_episode_number(self): if self.source.is_playlist: - sorted_media = Media.objects.filter(source=self.source) + sorted_media = Media.objects.filter( + source=self.source, + metadata__isnull=False, + ).order_by( + 'published', + 'created', + 'key', + ) else: - self_year = self.upload_date.year if self.upload_date else self.created.year - filtered_media = Media.objects.filter(source=self.source, published__year=self_year) - filtered_media = [m for m in filtered_media if m.upload_date is not None] - sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key)) - position_counter = 1 - for media in sorted_media: + self_year = self.created.year # unlikely to be accurate + if self.published: + self_year = self.published.year + elif self.has_metadata and self.upload_date: + self_year = self.upload_date.year + elif self.download_date: + # also, unlikely to be accurate + self_year = self.download_date.year + sorted_media = Media.objects.filter( + source=self.source, + metadata__isnull=False, + published__year=self_year, + ).order_by( + 'published', + 'created', + 'key', + ) + for counter, media in enumerate(sorted_media, start=1): if media == self: - return position_counter - position_counter += 1 + return counter def get_episode_str(self, use_padding=False): episode_number = self.calculate_episode_number()