diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 8f7afc2c..0c7507e9 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -7,9 +7,19 @@ import pstats import string import time from django.core.paginator import Paginator +from functools import partial +from operator import attrgetter, itemgetter +from pathlib import Path from urllib.parse import urlunsplit, urlencode, urlparse from .errors import DatabaseConnectionError +def directory_and_stem(arg_path, /, all_suffixes=False): + filepath = Path(arg_path) + stem = Path(filepath.stem) + while all_suffixes and stem.suffixes and '' != stem.suffix: + stem = Path(stem.stem) + return (filepath.parent, str(stem),) + def getenv(key, default=None, /, *, integer=False, string=True): ''' @@ -46,6 +56,51 @@ def getenv(key, default=None, /, *, integer=False, string=True): return r +def glob_quote(filestr, /): + _glob_specials = { + '?': '[?]', + '*': '[*]', + '[': '[[]', + ']': '[]]', # probably not needed, but it won't hurt + } + + if not isinstance(filestr, str): + raise TypeError(f'expected a str, got "{type(filestr)}"') + + return filestr.translate(str.maketrans(_glob_specials)) + + +def list_of_dictionaries(arg_list, /, arg_function=lambda x: x): + assert callable(arg_function) + if isinstance(arg_list, list): + _map_func = partial(lambda f, d: f(d) if isinstance(d, dict) else d, arg_function) + return (True, list(map(_map_func, arg_list)),) + return (False, arg_list,) + + +def mkdir_p(arg_path, /, *, mode=0o777): + ''' + Reminder: mode only affects the last directory + ''' + dirpath = Path(arg_path) + return dirpath.mkdir(mode=mode, parents=True, exist_ok=True) + + +def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): + result = list(iterable) + if key_func is None: + # itemgetter is the default + if item or not (item or attr): + key_func = itemgetter + elif attr: + key_func = attrgetter + for key, reverse in reversed(specs): + result.sort(key=key_func(key), reverse=reverse) + if use_reversed: + return list(reversed(result)) + return result + + def parse_database_connection_string(database_connection_string): ''' Parses a connection string in a URL style format, such as: @@ -167,6 +222,15 @@ def clean_emoji(s): return emoji.replace_emoji(s) +def seconds_to_timestr(seconds): + seconds = seconds % (24 * 3600) + hour = seconds // 3600 + seconds %= 3600 + minutes = seconds // 60 + seconds %= 60 + return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) + + def time_func(func): def wrapper(*args, **kwargs): start = time.perf_counter() diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 4196a9f8..f5fe3fd1 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -6,7 +6,7 @@ from .choices import Val, Fallback -from .utils import multi_key_sort +from common.utils import multi_key_sort from django.conf import settings diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 8cf41ce1..5ec14d7c 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,4 +1,3 @@ -from pathlib import Path from ..choices import Val, YouTube_SourceType # noqa @@ -11,11 +10,3 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2 element.tail = tail + (char * indent) return element -def directory_and_stem(arg_path, /, all_suffixes=False): - filepath = Path(arg_path) - stem = Path(filepath.stem) - while all_suffixes and stem.suffixes and '' != stem.suffix: - stem = Path(stem.stem) - stem = str(stem) - return (filepath.parent, stem,) - diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 62f73d5d..b2c79e15 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,15 +17,15 @@ from common.logger import log from common.errors import NoFormatException from common.json import JSONEncoder from common.utils import ( - clean_filename, clean_emoji, + clean_filename, clean_emoji, directory_and_stem, + glob_quote, mkdir_p, multi_key_sort, seconds_to_timestr, ) from ..youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, ) from ..utils import ( - seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, glob_quote, multi_key_sort, + filter_response, parse_media_format, write_text_file, ) from ..matching import ( get_best_combined_format, @@ -38,7 +38,7 @@ from ..choices import ( from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) -from ._private import _srctype_dict, _nfo_element, directory_and_stem +from ._private import _srctype_dict, _nfo_element from .media__tasks import ( download_checklist, download_finished, wait_for_premiere, ) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index d68a082f..f25d6a92 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log +from common.utils import glob_quote, mkdir_p from .models import Source, Media, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, @@ -17,7 +18,7 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media, download_source_images, delete_all_media_for_source, save_all_media_for_source, rename_media, get_media_metadata_task, get_media_download_task) -from .utils import delete_file, glob_quote, mkdir_p +from .utils import delete_file from .filtering import filter_media from .choices import Val, YouTube_SourceType diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index c5903bb0..f01f769e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -31,11 +31,11 @@ from common.errors import ( NoFormatException, NoMediaException, NoThumbnailException, DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, - remove_enclosed, ) + remove_enclosed, seconds_to_timestr, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, - write_text_file, filter_response, seconds_to_timestr, ) + write_text_file, filter_response, ) from .youtube import YouTubeError db_vendor = db.connection.vendor diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index fc7874fd..cbd14eab 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -2,11 +2,11 @@ import os import re import math from copy import deepcopy -from operator import attrgetter, itemgetter from pathlib import Path from tempfile import NamedTemporaryFile import requests from PIL import Image +from common.utils import list_of_dictionaries from django.conf import settings from urllib.parse import urlsplit, parse_qs from django.forms import ValidationError @@ -95,20 +95,6 @@ def resize_image_to_height(image, width, height): return image -def glob_quote(filestr): - _glob_specials = { - '?': '[?]', - '*': '[*]', - '[': '[[]', - ']': '[]]', # probably not needed, but it won't hurt - } - - if not isinstance(filestr, str): - raise TypeError(f'filestr must be a str, got "{type(filestr)}"') - - return filestr.translate(str.maketrans(_glob_specials)) - - def file_is_editable(filepath): ''' Checks that a file exists and the file is in an allowed predefined tuple of @@ -130,14 +116,6 @@ def file_is_editable(filepath): return False -def mkdir_p(arg_path, mode=0o777): - ''' - Reminder: mode only affects the last directory - ''' - dirpath = Path(arg_path) - return dirpath.mkdir(mode=mode, parents=True, exist_ok=True) - - def write_text_file(filepath, filedata): if not isinstance(filedata, str): raise TypeError(f'filedata must be a str, got "{type(filedata)}"') @@ -162,30 +140,6 @@ def delete_file(filepath): return False -def seconds_to_timestr(seconds): - seconds = seconds % (24 * 3600) - hour = seconds // 3600 - seconds %= 3600 - minutes = seconds // 60 - seconds %= 60 - return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) - - -def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): - result = list(iterable) - if key_func is None: - # itemgetter is the default - if item or not (item or attr): - key_func = itemgetter - elif attr: - key_func = attrgetter - for key, reverse in reversed(specs): - result.sort(key=key_func(key), reverse=reverse) - if use_reversed: - return list(reversed(result)) - return result - - def normalize_codec(codec_str): result = str(codec_str).upper() parts = result.split('.') @@ -201,17 +155,6 @@ def normalize_codec(codec_str): return result -def list_of_dictionaries(arg_list, arg_function=lambda x: x): - assert callable(arg_function) - if isinstance(arg_list, list): - def _call_func_with_dict(arg_dict): - if isinstance(arg_dict, dict): - return arg_function(arg_dict) - return arg_dict - return (True, list(map(_call_func_with_dict, arg_list)),) - return (False, arg_list,) - - def _url_keys(arg_dict, filter_func): result = {} if isinstance(arg_dict, dict): diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 5f13877f..493098cd 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -20,13 +20,13 @@ from django.utils._os import safe_join from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.timestamp import timestamp_to_datetime -from common.utils import append_uri_params +from common.utils import append_uri_params, mkdir_p, multi_key_sort from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, ConfirmDeleteMediaServerForm, SourceForm) -from .utils import validate_url, delete_file, multi_key_sort, mkdir_p +from .utils import delete_file, validate_url from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task, diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 7afdf337..f8516b69 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -7,6 +7,7 @@ import os from common.logger import log +from common.utils import mkdir_p from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory @@ -15,7 +16,6 @@ from urllib.parse import urlsplit, parse_qs from django.conf import settings from .choices import Val, FileExtension from .hooks import postprocessor_hook, progress_hook -from .utils import mkdir_p import yt_dlp import yt_dlp.patch.check_thumbnails import yt_dlp.patch.fatal_http_errors