Merge branch 'main' into patch-12

Dockerfile

@@ -25,6 +25,7 @@ ARG TARGETARCH
 
 ENV DEBIAN_FRONTEND="noninteractive" \
     APT_KEEP_ARCHIVES=1 \
+    EDITOR="editor" \
     HOME="/root" \
     LANGUAGE="en_US.UTF-8" \
     LANG="en_US.UTF-8" \
@@ -321,6 +322,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
   apt-get -y autoclean && \
   rm -v -f /var/cache/debconf/*.dat-old
 
+# The preference for openresty over nginx,
+# is for the newer version.
 FROM tubesync-openresty AS tubesync
 
 ARG S6_VERSION
@@ -343,16 +346,30 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
   # Install required distro packages
   apt-get -y --no-install-recommends install \
   libmariadb3 \
+  libonig5 \
   pkgconf \
   python3 \
   python3-libsass \
   python3-pip-whl \
   python3-socks \
   curl \
+  indent \
   less \
+  lua-lpeg \
+  tre-agrep \
+  vis \
+  xxd \
   && \
   # Link to the current python3 version
   ln -v -s -f -T "$(find /usr/local/lib -name 'python3.[0-9]*' -type d -printf '%P\n' | sort -r -V | head -n 1)" /usr/local/lib/python3 && \
+  # Configure the editor alternatives
+  touch /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \
+  update-alternatives --install /usr/bin/editor editor /usr/local/bin/babi 50 && \
+  update-alternatives --install /usr/local/bin/nano nano /bin/nano 10 && \
+  update-alternatives --install /usr/local/bin/nano nano /usr/local/bin/babi 20 && \
+  update-alternatives --install /usr/local/bin/vim vim /usr/bin/vim.tiny 15 && \
+  update-alternatives --install /usr/local/bin/vim vim /usr/bin/vis 35 && \
+  rm -v /usr/local/bin/babi /bin/nano /usr/bin/vim.tiny && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
@@ -404,6 +421,7 @@ RUN --mount=type=tmpfs,target=/cache \
   g++ \
   gcc \
   libjpeg-dev \
+  libonig-dev \
   libpq-dev \
   libwebp-dev \
   make \
@@ -447,6 +465,7 @@ RUN --mount=type=tmpfs,target=/cache \
   g++ \
   gcc \
   libjpeg-dev \
+  libonig-dev \
   libpq-dev \
   libwebp-dev \
   make \
@@ -456,8 +475,21 @@ RUN --mount=type=tmpfs,target=/cache \
   && \
   apt-get -y autopurge && \
   apt-get -y autoclean && \
+  LD_LIBRARY_PATH=/usr/local/lib/python3/dist-packages/pillow.libs:/usr/local/lib/python3/dist-packages/psycopg_binary.libs \
+    find /usr/local/lib/python3/dist-packages/ \
+      -name '*.so*' -print \
+      -exec du -h '{}' ';' \
+      -exec ldd '{}' ';' \
+    >| /cache/python-shared-objects 2>&1 && \
   rm -v -f /var/cache/debconf/*.dat-old && \
-  rm -v -rf /tmp/*
+  rm -v -rf /tmp/* ; \
+  if grep >/dev/null -Fe ' => not found' /cache/python-shared-objects ; \
+  then \
+      cat -v /cache/python-shared-objects ; \
+      printf -- 1>&2 '%s\n' \
+        ERROR: '    An unresolved shared object was found.' ; \
+      exit 1 ; \
+  fi
 
 # Copy root
 COPY config/root /

Pipfile

@@ -25,3 +25,4 @@ emoji = "*"
 brotli = "*"
 html5lib = "*"
 bgutil-ytdlp-pot-provider = "*"
+babi = "*"

tubesync/common/utils.py

@@ -7,9 +7,19 @@ import pstats
 import string
 import time
 from django.core.paginator import Paginator
+from functools import partial
+from operator import attrgetter, itemgetter
+from pathlib import Path
 from urllib.parse import urlunsplit, urlencode, urlparse
 from .errors import DatabaseConnectionError
 
+def directory_and_stem(arg_path, /, all_suffixes=False):
+    filepath = Path(arg_path)
+    stem = Path(filepath.stem)
+    while all_suffixes and stem.suffixes and '' != stem.suffix:
+        stem = Path(stem.stem)
+    return (filepath.parent, str(stem),)
+
 
 def getenv(key, default=None, /, *, integer=False, string=True):
     '''
@@ -46,6 +56,51 @@ def getenv(key, default=None, /, *, integer=False, string=True):
     return r
 
 
+def glob_quote(filestr, /):
+    _glob_specials = {
+        '?': '[?]',
+        '*': '[*]',
+        '[': '[[]',
+        ']': '[]]', # probably not needed, but it won't hurt
+    }
+
+    if not isinstance(filestr, str):
+        raise TypeError(f'expected a str, got "{type(filestr)}"')
+
+    return filestr.translate(str.maketrans(_glob_specials))
+
+
+def list_of_dictionaries(arg_list, /, arg_function=lambda x: x):
+    assert callable(arg_function)
+    if isinstance(arg_list, list):
+        _map_func = partial(lambda f, d: f(d) if isinstance(d, dict) else d, arg_function)
+        return (True, list(map(_map_func, arg_list)),)
+    return (False, arg_list,)
+
+
+def mkdir_p(arg_path, /, *, mode=0o777):
+    '''
+        Reminder: mode only affects the last directory
+    '''
+    dirpath = Path(arg_path)
+    return dirpath.mkdir(mode=mode, parents=True, exist_ok=True)
+
+
+def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None):
+    result = list(iterable)
+    if key_func is None:
+        # itemgetter is the default
+        if item or not (item or attr):
+            key_func = itemgetter
+        elif attr:
+            key_func = attrgetter
+    for key, reverse in reversed(specs):
+        result.sort(key=key_func(key), reverse=reverse)
+    if use_reversed:
+        return list(reversed(result))
+    return result
+
+
 def parse_database_connection_string(database_connection_string):
     '''
         Parses a connection string in a URL style format, such as:
@@ -167,6 +222,15 @@ def clean_emoji(s):
     return emoji.replace_emoji(s)
 
 
+def seconds_to_timestr(seconds):
+    seconds = seconds % (24 * 3600)
+    hour = seconds // 3600
+    seconds %= 3600
+    minutes = seconds // 60
+    seconds %= 60
+    return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
+
+
 def time_func(func):
     def wrapper(*args, **kwargs):
         start = time.perf_counter()

tubesync/sync/matching.py

@@ -6,7 +6,7 @@
 
 
 from .choices import Val, Fallback
-from .utils import multi_key_sort
+from common.utils import multi_key_sort
 from django.conf import settings
 
 

tubesync/sync/models/_private.py

@@ -1,4 +1,3 @@
-from pathlib import Path
 from ..choices import Val, YouTube_SourceType # noqa
 
 
@@ -11,11 +10,3 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2
     element.tail = tail + (char * indent)
     return element
 
-def directory_and_stem(arg_path, /, all_suffixes=False):
-    filepath = Path(arg_path)
-    stem = Path(filepath.stem)
-    while all_suffixes and stem.suffixes and '' != stem.suffix:
-        stem = Path(stem.stem)
-    stem = str(stem)
-    return (filepath.parent, stem,)
-

tubesync/sync/models/media.py

@@ -17,15 +17,15 @@ from common.logger import log
 from common.errors import NoFormatException
 from common.json import JSONEncoder
 from common.utils import (
-    clean_filename, clean_emoji,
+    clean_filename, clean_emoji, directory_and_stem,
+    glob_quote, mkdir_p, multi_key_sort, seconds_to_timestr,
 )
 from ..youtube import (
     get_media_info as get_youtube_media_info,
     download_media as download_youtube_media,
 )
 from ..utils import (
-    seconds_to_timestr, parse_media_format, filter_response,
+    filter_response, parse_media_format, write_text_file,
-    write_text_file, mkdir_p, glob_quote, multi_key_sort,
 )
 from ..matching import (
     get_best_combined_format,
@@ -38,7 +38,7 @@ from ..choices import (
 from ._migrations import (
     media_file_storage, get_media_thumb_path, get_media_file_path,
 )
-from ._private import _srctype_dict, _nfo_element, directory_and_stem
+from ._private import _srctype_dict, _nfo_element
 from .media__tasks import (
     download_checklist, download_finished, wait_for_premiere,
 )

tubesync/sync/signals.py

@@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _
 from background_task.signals import task_failed
 from background_task.models import Task
 from common.logger import log
+from common.utils import glob_quote, mkdir_p
 from .models import Source, Media, Metadata
 from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
                     download_media_thumbnail, download_media_metadata,
@@ -17,7 +18,7 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta
                     download_media, download_source_images,
                     delete_all_media_for_source, save_all_media_for_source,
                     rename_media, get_media_metadata_task, get_media_download_task)
-from .utils import delete_file, glob_quote, mkdir_p
+from .utils import delete_file
 from .filtering import filter_media
 from .choices import Val, YouTube_SourceType
 

tubesync/sync/tasks.py

@@ -31,11 +31,11 @@ from common.errors import ( NoFormatException, NoMediaException,
                             NoThumbnailException,
                             DownloadFailedException, )
 from common.utils import (  django_queryset_generator as qs_gen,
-                            remove_enclosed, )
+                            remove_enclosed, seconds_to_timestr, )
 from .choices import Val, TaskQueue
 from .models import Source, Media, MediaServer
 from .utils import ( get_remote_image, resize_image_to_height,
-                    write_text_file, filter_response, seconds_to_timestr, )
+                    write_text_file, filter_response, )
 from .youtube import YouTubeError
 
 db_vendor = db.connection.vendor

tubesync/sync/utils.py

@@ -2,11 +2,11 @@ import os
 import re
 import math
 from copy import deepcopy
-from operator import attrgetter, itemgetter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 import requests
 from PIL import Image
+from common.utils import list_of_dictionaries
 from django.conf import settings
 from urllib.parse import urlsplit, parse_qs
 from django.forms import ValidationError
@@ -95,20 +95,6 @@ def resize_image_to_height(image, width, height):
     return image
 
 
-def glob_quote(filestr):
-    _glob_specials = {
-        '?': '[?]',
-        '*': '[*]',
-        '[': '[[]',
-        ']': '[]]', # probably not needed, but it won't hurt
-    }
-
-    if not isinstance(filestr, str):
-        raise TypeError(f'filestr must be a str, got "{type(filestr)}"')
-
-    return filestr.translate(str.maketrans(_glob_specials))
-
-
 def file_is_editable(filepath):
     '''
         Checks that a file exists and the file is in an allowed predefined tuple of
@@ -130,14 +116,6 @@ def file_is_editable(filepath):
     return False
 
 
-def mkdir_p(arg_path, mode=0o777):
-    '''
-        Reminder: mode only affects the last directory
-    '''
-    dirpath = Path(arg_path)
-    return dirpath.mkdir(mode=mode, parents=True, exist_ok=True)
-
-
 def write_text_file(filepath, filedata):
     if not isinstance(filedata, str):
         raise TypeError(f'filedata must be a str, got "{type(filedata)}"')
@@ -162,30 +140,6 @@ def delete_file(filepath):
     return False
 
 
-def seconds_to_timestr(seconds):
-   seconds = seconds % (24 * 3600)
-   hour = seconds // 3600
-   seconds %= 3600
-   minutes = seconds // 60
-   seconds %= 60
-   return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
-
-
-def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None):
-    result = list(iterable)
-    if key_func is None:
-        # itemgetter is the default
-        if item or not (item or attr):
-            key_func = itemgetter
-        elif attr:
-            key_func = attrgetter
-    for key, reverse in reversed(specs):
-        result.sort(key=key_func(key), reverse=reverse)
-    if use_reversed:
-        return list(reversed(result))
-    return result
-
-
 def normalize_codec(codec_str):
     result = str(codec_str).upper()
     parts = result.split('.')
@@ -201,17 +155,6 @@ def normalize_codec(codec_str):
     return result
 
 
-def list_of_dictionaries(arg_list, arg_function=lambda x: x):
-    assert callable(arg_function)
-    if isinstance(arg_list, list):
-        def _call_func_with_dict(arg_dict):
-            if isinstance(arg_dict, dict):
-                return arg_function(arg_dict)
-            return arg_dict
-        return (True, list(map(_call_func_with_dict, arg_list)),)
-    return (False, arg_list,)
-
-
 def _url_keys(arg_dict, filter_func):
     result = {}
     if isinstance(arg_dict, dict):

tubesync/sync/views.py

@@ -20,13 +20,13 @@ from django.utils._os import safe_join
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.timestamp import timestamp_to_datetime
-from common.utils import append_uri_params
+from common.utils import append_uri_params, mkdir_p, multi_key_sort
 from background_task.models import Task, CompletedTask
 from .models import Source, Media, MediaServer
 from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
                     SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm,
                     ConfirmDeleteMediaServerForm, SourceForm)
-from .utils import validate_url, delete_file, multi_key_sort, mkdir_p
+from .utils import delete_file, validate_url
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
                     delete_task_by_media, index_source_task,

tubesync/sync/youtube.py

@@ -7,6 +7,7 @@
 import os
 
 from common.logger import log
+from common.utils import mkdir_p
 from copy import deepcopy
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -15,7 +16,6 @@ from urllib.parse import urlsplit, parse_qs
 from django.conf import settings
 from .choices import Val, FileExtension
 from .hooks import postprocessor_hook, progress_hook
-from .utils import mkdir_p
 import yt_dlp
 import yt_dlp.patch.check_thumbnails
 import yt_dlp.patch.fatal_http_errors