From aedaa6f7b748cb3ba83ac6238759de9f6530942d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 03:33:33 -0400 Subject: [PATCH 01/35] Copy generic functions from `sync.utils` --- tubesync/common/utils.py | 65 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index c4798943..fdf04352 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -8,11 +8,22 @@ import string import time from datetime import datetime from django.core.paginator import Paginator +from functools import partial +from operator import attrgetter, itemgetter +from pathlib import Path from urllib.parse import urlunsplit, urlencode, urlparse from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError +def directory_and_stem(arg_path, /): + filepath = Path(arg_path) + stem = Path(filepath.stem) + while stem.suffixes and '' != stem.suffix: + stem = Path(stem.stem) + return (filepath.parent, str(stem),) + + def getenv(key, default=None, /, *, integer=False, string=True): ''' Guarantees a returned type from calling `os.getenv` @@ -48,6 +59,51 @@ def getenv(key, default=None, /, *, integer=False, string=True): return r +def glob_quote(filestr, /): + _glob_specials = { + '?': '[?]', + '*': '[*]', + '[': '[[]', + ']': '[]]', # probably not needed, but it won't hurt + } + + if not isinstance(filestr, str): + raise TypeError(f'expected a str, got "{type(filestr)}"') + + return filestr.translate(str.maketrans(_glob_specials)) + + +def list_of_dictionaries(arg_list, /, *, arg_function=lambda x: x): + assert callable(arg_function) + if isinstance(arg_list, list): + _map_func = partial(lambda f, d: f(d) if isinstance(d, dict) else d, arg_function) + return (True, list(map(_map_func, arg_list)),) + return (False, arg_list,) + + +def mkdir_p(arg_path, /, *, mode=0o777): + ''' + Reminder: mode only affects the last directory + ''' + dirpath = Path(arg_path) + return dirpath.mkdir(mode=mode, parents=True, exist_ok=True) + + +def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): + result = list(iterable) + if key_func is None: + # itemgetter is the default + if item or not (item or attr): + key_func = itemgetter + elif attr: + key_func = attrgetter + for key, reverse in reversed(specs): + result.sort(key=key_func(key), reverse=reverse) + if use_reversed: + return list(reversed(result)) + return result + + def parse_database_connection_string(database_connection_string): ''' Parses a connection string in a URL style format, such as: @@ -180,6 +236,15 @@ def json_serial(obj): raise TypeError(f'Type {type(obj)} is not json_serial()-able') +def seconds_to_timestr(seconds): + seconds = seconds % (24 * 3600) + hour = seconds // 3600 + seconds %= 3600 + minutes = seconds // 60 + seconds %= 60 + return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) + + def time_func(func): def wrapper(*args, **kwargs): start = time.perf_counter() From 3aa5ec9aae4a9f7caa80586a17428ba76df200ea Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 04:25:47 -0400 Subject: [PATCH 02/35] Remove `json_serial` as it is in json.py instead --- tubesync/common/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 64c597cb..e773bb5b 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -223,14 +223,6 @@ def clean_emoji(s): return emoji.replace_emoji(s) -def json_serial(obj): - if isinstance(obj, datetime): - return obj.isoformat() - if isinstance(obj, LazyList): - return list(obj) - raise TypeError(f'Type {type(obj)} is not json_serial()-able') - - def seconds_to_timestr(seconds): seconds = seconds % (24 * 3600) hour = seconds // 3600 From 360b39ebcf166059d72f4cc7a94825cf17be76e3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 24 May 2025 23:03:45 -0400 Subject: [PATCH 03/35] Update `directory_and_stem` in utils.py --- tubesync/common/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index e773bb5b..7256b3d8 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -13,11 +13,10 @@ from pathlib import Path from urllib.parse import urlunsplit, urlencode, urlparse from .errors import DatabaseConnectionError - -def directory_and_stem(arg_path, /): +def directory_and_stem(arg_path, /, all_suffixes=False): filepath = Path(arg_path) stem = Path(filepath.stem) - while stem.suffixes and '' != stem.suffix: + while all_suffixes and stem.suffixes and '' != stem.suffix: stem = Path(stem.stem) return (filepath.parent, str(stem),) From 1ac70a16a0e0442371c71c893b467c006a160aff Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 01:56:18 -0400 Subject: [PATCH 04/35] Enable `editor` in the container For less than 3 MiB, I can have a modern editor that does enough of what `vim-nox` does that zi won't miss that. --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index da1e84c5..aba92a07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,6 +321,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old +# The preference for openresty over nginx, +# is for the newer version. FROM tubesync-openresty AS tubesync ARG S6_VERSION @@ -352,7 +354,12 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va python3-pip-whl \ python3-socks \ curl \ + indent \ less \ + lua-lpeg \ + tre-agrep \ + vis \ + xxd \ && \ # Link to the current python3 version ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \ From 89abf8f56875c898de15d5fdb16c7f7018b041ae Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 02:01:02 -0400 Subject: [PATCH 05/35] Add `babi` editor --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 49b5127b..34a52aa8 100644 --- a/Pipfile +++ b/Pipfile @@ -25,3 +25,4 @@ emoji = "*" brotli = "*" html5lib = "*" bgutil-ytdlp-pot-provider = "*" +babi = "*" From e5c19a2c47984af152e5ffa60d336064b8a3a2ec Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 12:39:10 -0400 Subject: [PATCH 06/35] Configure alternatives groups --- Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aba92a07..735c10eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -362,7 +362,14 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va xxd \ && \ # Link to the current python3 version - ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \ + update-alternatives --install /usr/local/lib/python3 python3-lib \ + "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" 100 && \ + # Configure the editor and nano alternatives + touch /usr/local/bin/babi /bin/nano && \ + update-alternatives --install /usr/local/bin/nano nano /bin/nano 10 && \ + update-alternatives --install /usr/local/bin/nano nano /usr/local/bin/babi 20 && \ + update-alternatives --install /usr/bin/editor editor /usr/local/bin/babi 50 && \ + rm -v /usr/local/bin/babi /bin/nano && \ # Create a 'app' user which the application will run as groupadd app && \ useradd -M -d /app -s /bin/false -g app app && \ From c1e30b775117d6046dfec32853358c08e84c7d76 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 12:57:21 -0400 Subject: [PATCH 07/35] Add the `vim` alternatives group --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 735c10eb..d2df5d67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -364,12 +364,14 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va # Link to the current python3 version update-alternatives --install /usr/local/lib/python3 python3-lib \ "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" 100 && \ - # Configure the editor and nano alternatives - touch /usr/local/bin/babi /bin/nano && \ + # Configure the editor alternatives + touch /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \ + update-alternatives --install /usr/bin/editor editor /usr/local/bin/babi 50 && \ update-alternatives --install /usr/local/bin/nano nano /bin/nano 10 && \ update-alternatives --install /usr/local/bin/nano nano /usr/local/bin/babi 20 && \ - update-alternatives --install /usr/bin/editor editor /usr/local/bin/babi 50 && \ - rm -v /usr/local/bin/babi /bin/nano && \ + update-alternatives --install /usr/local/bin/vim vim /usr/bin/vim.tiny 15 && \ + rm -v /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \ + update-alternatives --install /usr/local/bin/vim vim /usr/bin/vis 35 && \ # Create a 'app' user which the application will run as groupadd app && \ useradd -M -d /app -s /bin/false -g app app && \ From 8b0d3ebd30b323648ff14d72d8bc517ee5a305b0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 13:00:21 -0400 Subject: [PATCH 08/35] Set the `EDITOR` environment variable to use the alternatives group --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index d2df5d67..022a2b7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" \ APT_KEEP_ARCHIVES=1 \ + EDITOR="editor" \ HOME="/root" \ LANGUAGE="en_US.UTF-8" \ LANG="en_US.UTF-8" \ From 133d65a6c98b02685b966ca51384b72c1f9cdfe4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 1 Jun 2025 13:04:18 -0400 Subject: [PATCH 09/35] Prevent `update-alternatives` from doing the wrong thing --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 022a2b7b..7ad61039 100644 --- a/Dockerfile +++ b/Dockerfile @@ -371,8 +371,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va update-alternatives --install /usr/local/bin/nano nano /bin/nano 10 && \ update-alternatives --install /usr/local/bin/nano nano /usr/local/bin/babi 20 && \ update-alternatives --install /usr/local/bin/vim vim /usr/bin/vim.tiny 15 && \ - rm -v /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \ update-alternatives --install /usr/local/bin/vim vim /usr/bin/vis 35 && \ + rm -v /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \ # Create a 'app' user which the application will run as groupadd app && \ useradd -M -d /app -s /bin/false -g app app && \ From 6d04d773c3bedfe5ed2cde907526ef1488f5f8a5 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 14:39:07 -0400 Subject: [PATCH 10/35] Remove `glob_quote` from sync/utils.py --- tubesync/sync/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index fc7874fd..46668c25 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -95,20 +95,6 @@ def resize_image_to_height(image, width, height): return image -def glob_quote(filestr): - _glob_specials = { - '?': '[?]', - '*': '[*]', - '[': '[[]', - ']': '[]]', # probably not needed, but it won't hurt - } - - if not isinstance(filestr, str): - raise TypeError(f'filestr must be a str, got "{type(filestr)}"') - - return filestr.translate(str.maketrans(_glob_specials)) - - def file_is_editable(filepath): ''' Checks that a file exists and the file is in an allowed predefined tuple of From dbd30e65d6a4b96316b1561b6795785aeefd0573 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 14:42:12 -0400 Subject: [PATCH 11/35] Adjust the `glob_quote` import in sync/signals.py --- tubesync/sync/signals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index d68a082f..e1a22ee5 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log +from common.utils import glob_quote from .models import Source, Media, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, @@ -17,7 +18,7 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media, download_source_images, delete_all_media_for_source, save_all_media_for_source, rename_media, get_media_metadata_task, get_media_download_task) -from .utils import delete_file, glob_quote, mkdir_p +from .utils import delete_file, mkdir_p from .filtering import filter_media from .choices import Val, YouTube_SourceType From c7e8c974e67e6ed49f225c44411d5e7937d064b6 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 14:45:28 -0400 Subject: [PATCH 12/35] Adjust the `glob_quote` import in sync/media.py --- tubesync/sync/models/media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 62f73d5d..32e026e8 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,7 +17,7 @@ from common.logger import log from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( - clean_filename, clean_emoji, + clean_filename, clean_emoji, glob_quote, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -25,7 +25,7 @@ from ..youtube import ( ) from ..utils import ( seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, glob_quote, multi_key_sort, + write_text_file, mkdir_p, multi_key_sort, ) from ..matching import ( get_best_combined_format, From 7b23d3b0f01567df99408fac5052ceb5510421d8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 14:51:45 -0400 Subject: [PATCH 13/35] Remove `seconds_to_timestr` from sync/utils.py --- tubesync/sync/utils.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 46668c25..3f62492e 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -148,15 +148,6 @@ def delete_file(filepath): return False -def seconds_to_timestr(seconds): - seconds = seconds % (24 * 3600) - hour = seconds // 3600 - seconds %= 3600 - minutes = seconds // 60 - seconds %= 60 - return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) - - def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): result = list(iterable) if key_func is None: From 2c67b2f31c4b1f0dd118c47b1e99c914a39a77e2 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 14:59:08 -0400 Subject: [PATCH 14/35] Adjust the `seconds_to_timestr` import in sync/tasks.py --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index c5903bb0..f01f769e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -31,11 +31,11 @@ from common.errors import ( NoFormatException, NoMediaException, NoThumbnailException, DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, - remove_enclosed, ) + remove_enclosed, seconds_to_timestr, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, - write_text_file, filter_response, seconds_to_timestr, ) + write_text_file, filter_response, ) from .youtube import YouTubeError db_vendor = db.connection.vendor From db7af44cec1b8a9059969748157f1222ffdaa270 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:03:32 -0400 Subject: [PATCH 15/35] Adjust the `seconds_to_timestr` import in sync/media.py --- tubesync/sync/models/media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 32e026e8..f50de356 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -18,13 +18,14 @@ from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( clean_filename, clean_emoji, glob_quote, + seconds_to_timestr, ) from ..youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, ) from ..utils import ( - seconds_to_timestr, parse_media_format, filter_response, + parse_media_format, filter_response, write_text_file, mkdir_p, multi_key_sort, ) from ..matching import ( From 8852c8c2eff06ec916e653d033cf5af5ea4d1c98 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:16:47 -0400 Subject: [PATCH 16/35] Switch to the `common.utils` version --- tubesync/sync/models/media.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index f50de356..377d87c3 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,8 +17,8 @@ from common.logger import log from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( - clean_filename, clean_emoji, glob_quote, - seconds_to_timestr, + clean_filename, clean_emoji, directory_and_stem, + glob_quote, seconds_to_timestr, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -39,7 +39,7 @@ from ..choices import ( from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) -from ._private import _srctype_dict, _nfo_element, directory_and_stem +from ._private import _srctype_dict, _nfo_element from .media__tasks import ( download_checklist, download_finished, wait_for_premiere, ) From 8b127afc234f1bddca5198aea53b022c78767b06 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:18:57 -0400 Subject: [PATCH 17/35] Remove the old copy from sync/models/_private.py --- tubesync/sync/models/_private.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 8cf41ce1..094d3763 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -11,11 +11,3 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2 element.tail = tail + (char * indent) return element -def directory_and_stem(arg_path, /, all_suffixes=False): - filepath = Path(arg_path) - stem = Path(filepath.stem) - while all_suffixes and stem.suffixes and '' != stem.suffix: - stem = Path(stem.stem) - stem = str(stem) - return (filepath.parent, stem,) - From 432bc019d5197b3adcdfaaabc924e533a3e99268 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:23:18 -0400 Subject: [PATCH 18/35] fixup: remove an unused import --- tubesync/sync/models/_private.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 094d3763..5ec14d7c 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,4 +1,3 @@ -from pathlib import Path from ..choices import Val, YouTube_SourceType # noqa From dc0edef5624d1dc0bc8c0231c9eb3790a42275d2 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:31:32 -0400 Subject: [PATCH 19/35] Remove `list_of_dictionaries` from sync/utils.py --- tubesync/sync/utils.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 3f62492e..617bccca 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -7,6 +7,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile import requests from PIL import Image +from common.utils import list_of_dictionaries from django.conf import settings from urllib.parse import urlsplit, parse_qs from django.forms import ValidationError @@ -178,17 +179,6 @@ def normalize_codec(codec_str): return result -def list_of_dictionaries(arg_list, arg_function=lambda x: x): - assert callable(arg_function) - if isinstance(arg_list, list): - def _call_func_with_dict(arg_dict): - if isinstance(arg_dict, dict): - return arg_function(arg_dict) - return arg_dict - return (True, list(map(_call_func_with_dict, arg_list)),) - return (False, arg_list,) - - def _url_keys(arg_dict, filter_func): result = {} if isinstance(arg_dict, dict): From f097b6b4e06e33db0b3b5f9840bc6d64d3791ba4 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:38:36 -0400 Subject: [PATCH 20/35] Remove `mkdir_p` from sync/utils.py --- tubesync/sync/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 617bccca..9b18248b 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -117,14 +117,6 @@ def file_is_editable(filepath): return False -def mkdir_p(arg_path, mode=0o777): - ''' - Reminder: mode only affects the last directory - ''' - dirpath = Path(arg_path) - return dirpath.mkdir(mode=mode, parents=True, exist_ok=True) - - def write_text_file(filepath, filedata): if not isinstance(filedata, str): raise TypeError(f'filedata must be a str, got "{type(filedata)}"') From 0be0aa9e812cf1acc74a68bbf40a4352ccc1af77 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:46:50 -0400 Subject: [PATCH 21/35] Adjust the `mkdir_p` import in sync/models/media.py --- tubesync/sync/models/media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 377d87c3..ab21fb9f 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -18,7 +18,7 @@ from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( clean_filename, clean_emoji, directory_and_stem, - glob_quote, seconds_to_timestr, + glob_quote, mkdir_p, seconds_to_timestr, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -26,7 +26,7 @@ from ..youtube import ( ) from ..utils import ( parse_media_format, filter_response, - write_text_file, mkdir_p, multi_key_sort, + write_text_file, multi_key_sort, ) from ..matching import ( get_best_combined_format, From ff30543a2016a78bfac6da4b59a03395b21cce55 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:55:58 -0400 Subject: [PATCH 22/35] Adjust the `mkdir_p` import in sync/signals.py --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index e1a22ee5..f25d6a92 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -10,7 +10,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from common.utils import glob_quote +from common.utils import glob_quote, mkdir_p from .models import Source, Media, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, @@ -18,7 +18,7 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media, download_source_images, delete_all_media_for_source, save_all_media_for_source, rename_media, get_media_metadata_task, get_media_download_task) -from .utils import delete_file, mkdir_p +from .utils import delete_file from .filtering import filter_media from .choices import Val, YouTube_SourceType From fbb27eaa5dce8e3d351c6a50b47e21c6827e5c30 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 15:58:11 -0400 Subject: [PATCH 23/35] Adjust the `mkdir_p` import in sync/views.py --- tubesync/sync/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 5f13877f..45236465 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -20,13 +20,13 @@ from django.utils._os import safe_join from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.timestamp import timestamp_to_datetime -from common.utils import append_uri_params +from common.utils import append_uri_params, mkdir_p from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, ConfirmDeleteMediaServerForm, SourceForm) -from .utils import validate_url, delete_file, multi_key_sort, mkdir_p +from .utils import validate_url, delete_file, multi_key_sort from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task, From 021988ff9ffcc0e8009af4a82dbe625f23773521 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 16:02:12 -0400 Subject: [PATCH 24/35] Adjust the `mkdir_p` import in sync/youtube.py --- tubesync/sync/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 7afdf337..f8516b69 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -7,6 +7,7 @@ import os from common.logger import log +from common.utils import mkdir_p from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory @@ -15,7 +16,6 @@ from urllib.parse import urlsplit, parse_qs from django.conf import settings from .choices import Val, FileExtension from .hooks import postprocessor_hook, progress_hook -from .utils import mkdir_p import yt_dlp import yt_dlp.patch.check_thumbnails import yt_dlp.patch.fatal_http_errors From 1afe0d41539731fb7d16fd5969d8b825b5096a03 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 18:31:48 -0400 Subject: [PATCH 25/35] Adjust the `multi_key_sort` import in sync/matching.py --- tubesync/sync/matching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 4196a9f8..f5fe3fd1 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -6,7 +6,7 @@ from .choices import Val, Fallback -from .utils import multi_key_sort +from common.utils import multi_key_sort from django.conf import settings From 64cd082406ad3e6baf7887af6069e59c71275cb2 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 18:40:05 -0400 Subject: [PATCH 26/35] Remove `multi_key_sort` from sync/utils.py --- tubesync/sync/utils.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 9b18248b..7c8947bb 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -141,21 +141,6 @@ def delete_file(filepath): return False -def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): - result = list(iterable) - if key_func is None: - # itemgetter is the default - if item or not (item or attr): - key_func = itemgetter - elif attr: - key_func = attrgetter - for key, reverse in reversed(specs): - result.sort(key=key_func(key), reverse=reverse) - if use_reversed: - return list(reversed(result)) - return result - - def normalize_codec(codec_str): result = str(codec_str).upper() parts = result.split('.') From decbe14968a5ce43df89d5f67d10345d64034226 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 18:47:52 -0400 Subject: [PATCH 27/35] Adjust the `multi_key_sort` import in sync/models/media.py --- tubesync/sync/models/media.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index ab21fb9f..b2c79e15 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -18,15 +18,14 @@ from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( clean_filename, clean_emoji, directory_and_stem, - glob_quote, mkdir_p, seconds_to_timestr, + glob_quote, mkdir_p, multi_key_sort, seconds_to_timestr, ) from ..youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, ) from ..utils import ( - parse_media_format, filter_response, - write_text_file, multi_key_sort, + filter_response, parse_media_format, write_text_file, ) from ..matching import ( get_best_combined_format, From ae1974b503c734c1b220a4f577f00cc6c101bdec Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 18:52:09 -0400 Subject: [PATCH 28/35] Adjust the `multi_key_sort` import in sync/views.py --- tubesync/sync/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 45236465..493098cd 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -20,13 +20,13 @@ from django.utils._os import safe_join from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.timestamp import timestamp_to_datetime -from common.utils import append_uri_params, mkdir_p +from common.utils import append_uri_params, mkdir_p, multi_key_sort from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, ConfirmDeleteMediaServerForm, SourceForm) -from .utils import validate_url, delete_file, multi_key_sort +from .utils import delete_file, validate_url from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task, From 659e46a31f85535f6d199491958ec806e63719e8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 18:55:36 -0400 Subject: [PATCH 29/35] fixup: remove unused imports --- tubesync/sync/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 7c8947bb..cbd14eab 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -2,7 +2,6 @@ import os import re import math from copy import deepcopy -from operator import attrgetter, itemgetter from pathlib import Path from tempfile import NamedTemporaryFile import requests From 60aafbdab4c7f4b770be3c0b9ce77f8280b02757 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Jun 2025 19:04:40 -0400 Subject: [PATCH 30/35] fixup: accept the existing calls --- tubesync/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 7256b3d8..0c7507e9 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -70,7 +70,7 @@ def glob_quote(filestr, /): return filestr.translate(str.maketrans(_glob_specials)) -def list_of_dictionaries(arg_list, /, *, arg_function=lambda x: x): +def list_of_dictionaries(arg_list, /, arg_function=lambda x: x): assert callable(arg_function) if isinstance(arg_list, list): _map_func = partial(lambda f, d: f(d) if isinstance(d, dict) else d, arg_function) From 3fc2b09a26f6cfa69630863e00b39e262684b75f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 3 Jun 2025 09:56:15 -0400 Subject: [PATCH 31/35] Remove the `python3-lib` alternative --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ad61039..ffc08066 100644 --- a/Dockerfile +++ b/Dockerfile @@ -363,8 +363,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va xxd \ && \ # Link to the current python3 version - update-alternatives --install /usr/local/lib/python3 python3-lib \ - "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" 100 && \ + ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \ # Configure the editor alternatives touch /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \ update-alternatives --install /usr/bin/editor editor /usr/local/bin/babi 50 && \ From f74c22c849a216cc807a97e2d3930349e6044133 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 3 Jun 2025 10:32:28 -0400 Subject: [PATCH 32/35] fixup: building from source on arm64 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index ffc08066..2dee66ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -423,6 +423,7 @@ RUN --mount=type=tmpfs,target=/cache \ g++ \ gcc \ libjpeg-dev \ + libonig-dev \ libpq-dev \ libwebp-dev \ make \ @@ -466,6 +467,7 @@ RUN --mount=type=tmpfs,target=/cache \ g++ \ gcc \ libjpeg-dev \ + libonig-dev \ libpq-dev \ libwebp-dev \ make \ From a849367ac86dc75955dc1396f6647941fe6c3ad1 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 3 Jun 2025 10:48:17 -0400 Subject: [PATCH 33/35] Add the library `libonig-dev` depends upon also --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2dee66ac..2a95f304 100644 --- a/Dockerfile +++ b/Dockerfile @@ -347,6 +347,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y --no-install-recommends install \ libjpeg62-turbo \ libmariadb3 \ + libonig5 \ libpq5 \ libwebp7 \ pkgconf \ From 8baf50d98e98db7148a34e0f78ff6c3051103602 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 3 Jun 2025 13:09:56 -0400 Subject: [PATCH 34/35] Check for unresolved Python shared libraries --- Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index da1e84c5..61324a9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -459,8 +459,16 @@ RUN --mount=type=tmpfs,target=/cache \ && \ apt-get -y autopurge && \ apt-get -y autoclean && \ + LD_LIBRARY_PATH=/usr/local/lib/python3/dist-packages/pillow.libs:/usr/local/lib/python3/dist-packages/psycopg_binary.libs \ + find /usr/local/lib/python3/dist-packages/ \ + -name '*.so*' -print \ + -exec du -h '{}' ';' \ + -exec ldd '{}' ';' \ + >| /cache/python-shared-objects 2>&1 && \ rm -v -f /var/cache/debconf/*.dat-old && \ - rm -v -rf /tmp/* + rm -v -rf /tmp/* ; \ + grep >/dev/null -Fe ' => not found' /cache/python-shared-objects && \ + cat -v /cache/python-shared-objects || : # Copy root COPY config/root / From 75aa06d495ab6cfb73a28d74a4fdcf609721aab2 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 3 Jun 2025 13:34:25 -0400 Subject: [PATCH 35/35] Stop the build when a shared object is unresolved --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61324a9d..353a54ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -467,8 +467,13 @@ RUN --mount=type=tmpfs,target=/cache \ >| /cache/python-shared-objects 2>&1 && \ rm -v -f /var/cache/debconf/*.dat-old && \ rm -v -rf /tmp/* ; \ - grep >/dev/null -Fe ' => not found' /cache/python-shared-objects && \ - cat -v /cache/python-shared-objects || : + if grep >/dev/null -Fe ' => not found' /cache/python-shared-objects ; \ + then \ + cat -v /cache/python-shared-objects ; \ + printf -- 1>&2 '%s\n' \ + ERROR: ' An unresolved shared object was found.' ; \ + exit 1 ; \ + fi # Copy root COPY config/root /