diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index fd599d06..944467fc 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -14,12 +14,14 @@ from django.core.validators import RegexValidator from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from common.logger import log from common.errors import NoFormatException from common.utils import clean_filename, clean_emoji from .youtube import (get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) -from .utils import seconds_to_timestr, parse_media_format, filter_response +from .utils import (seconds_to_timestr, parse_media_format, filter_response, + write_text_file, mkdir_p, directory_and_stem, glob_quote) from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from .mediaservers import PlexMediaServer @@ -1566,6 +1568,79 @@ class Media(models.Model): return str(episode_number) + def rename_files(self): + if self.downloaded and self.media_file: + old_video_path = Path(self.media_file.path) + new_video_path = Path(get_media_file_path(self, None)) + if old_video_path.exists() and not new_video_path.exists(): + old_video_path = old_video_path.resolve(strict=True) + + # move video to destination + mkdir_p(new_video_path.parent) + log.debug(f'{self!s}: {old_video_path!s} => {new_video_path!s}') + old_video_path.rename(new_video_path) + log.info(f'Renamed video file for: {self!s}') + + # collect the list of files to move + # this should not include the video we just moved + (old_prefix_path, old_stem) = directory_and_stem(old_video_path) + other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) + log.info(f'Collected {len(other_paths)} other paths for: {self!s}') + + # adopt orphaned files, if possible + media_format = str(self.source.media_format) + top_dir_path = Path(self.source.directory_path) + if '{key}' in media_format: + fuzzy_paths = list(top_dir_path.rglob('*' + glob_quote(str(self.key)) + '*')) + log.info(f'Collected {len(fuzzy_paths)} fuzzy paths for: {self!s}') + + if new_video_path.exists(): + new_video_path = new_video_path.resolve(strict=True) + + # update the media_file in the db + self.media_file.name = str(new_video_path.relative_to(self.media_file.storage.location)) + self.save() + log.info(f'Updated "media_file" in the database for: {self!s}') + + (new_prefix_path, new_stem) = directory_and_stem(new_video_path) + + # move and change names to match stem + for other_path in other_paths: + old_file_str = other_path.name + new_file_str = new_stem + old_file_str[len(old_stem):] + new_file_path = Path(new_prefix_path / new_file_str) + log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}') + # it should exist, but check anyway + if other_path.exists(): + log.debug(f'{self!s}: {other_path!s} => {new_file_path!s}') + other_path.replace(new_file_path) + + for fuzzy_path in fuzzy_paths: + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path) + old_file_str = fuzzy_path.name + new_file_str = new_stem + old_file_str[len(fuzzy_stem):] + new_file_path = Path(new_prefix_path / new_file_str) + log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}') + # it quite possibly was renamed already + if fuzzy_path.exists() and not new_file_path.exists(): + log.debug(f'{self!s}: {fuzzy_path!s} => {new_file_path!s}') + fuzzy_path.rename(new_file_path) + + # The thumbpath inside the .nfo file may have changed + if self.source.write_nfo and self.source.copy_thumbnails: + write_text_file(new_prefix_path / self.nfopath.name, self.nfoxml) + log.info(f'Wrote new ".nfo" file for: {self!s}') + + # try to remove empty dirs + parent_dir = old_video_path.parent + try: + while parent_dir.is_dir(): + parent_dir.rmdir() + log.info(f'Removed empty directory: {parent_dir!s}') + parent_dir = parent_dir.parent + except OSError as e: + pass + class MediaServer(models.Model): ''' diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index e2a1398c..92b1e7ec 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -12,7 +12,8 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, download_media, rescan_media_server, download_source_images, - save_all_media_for_source, get_media_metadata_task) + save_all_media_for_source, rename_all_media_for_source, + get_media_metadata_task) from .utils import delete_file from .filtering import filter_media @@ -53,7 +54,7 @@ def source_post_save(sender, instance, created, **kwargs): if instance.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST and instance.copy_channel_images: download_source_images( str(instance.pk), - priority=0, + priority=2, verbose_name=verbose_name.format(instance.name) ) if instance.index_schedule > 0: @@ -68,10 +69,28 @@ def source_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) + # Check settings before any rename tasks are scheduled + rename_sources_setting = settings.RENAME_SOURCES or list() + create_rename_task = ( + ( + instance.directory and + instance.directory in rename_sources_setting + ) or + settings.RENAME_ALL_SOURCES + ) + if create_rename_task: + verbose_name = _('Renaming all media for source "{}"') + rename_all_media_for_source( + str(instance.pk), + queue=str(instance.pk), + priority=1, + verbose_name=verbose_name.format(instance.name), + remove_existing_tasks=False + ) verbose_name = _('Checking all media for source "{}"') save_all_media_for_source( str(instance.pk), - priority=0, + priority=2, verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 8d52b3bf..d677df40 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -51,6 +51,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, + 'sync.tasks.rename_all_media_for_source': Source, } MODEL_URL_MAP = { Source: 'sync:source', @@ -516,3 +517,18 @@ def save_all_media_for_source(source_id): # flags may need to be recalculated for media in Media.objects.filter(source=source): media.save() + + +@background(schedule=0) +def rename_all_media_for_source(source_id): + try: + source = Source.objects.get(pk=source_id) + except Source.DoesNotExist: + # Task triggered but the source no longer exists, do nothing + log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no ' + f'source exists with ID: {source_id}') + return + for media in Media.objects.filter(source=source): + media.rename_files() + + diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index f0bef5c5..2f9e6593 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -94,6 +94,20 @@ def resize_image_to_height(image, width, height): return image +def glob_quote(filestr): + _glob_specials = { + '?': '[?]', + '*': '[*]', + '[': '[[]', + ']': '[]]', # probably not needed, but it won't hurt + } + + if not isinstance(filestr, str): + raise TypeError(f'filestr must be a str, got "{type(filestr)}"') + + return filestr.translate(str.maketrans(_glob_specials)) + + def file_is_editable(filepath): ''' Checks that a file exists and the file is in an allowed predefined tuple of @@ -115,6 +129,23 @@ def file_is_editable(filepath): return False +def directory_and_stem(arg_path): + filepath = Path(arg_path) + stem = Path(filepath.stem) + while stem.suffixes and '' != stem.suffix: + stem = Path(stem.stem) + stem = str(stem) + return (filepath.parent, stem,) + + +def mkdir_p(arg_path, mode=0o777): + ''' + Reminder: mode only affects the last directory + ''' + dirpath = Path(arg_path) + return dirpath.mkdir(mode=mode, parents=True, exist_ok=True) + + def write_text_file(filepath, filedata): if not isinstance(filedata, str): raise TypeError(f'filedata must be a str, got "{type(filedata)}"') diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index e2138f9a..9756f924 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -9,6 +9,7 @@ from pathlib import Path from django.conf import settings from copy import copy from common.logger import log +from .utils import mkdir_p import yt_dlp @@ -21,7 +22,7 @@ _youtubedl_tempdir = getattr(settings, 'YOUTUBE_DL_TEMPDIR', None) if _youtubedl_tempdir: _youtubedl_tempdir = str(_youtubedl_tempdir) _youtubedl_tempdir_path = Path(_youtubedl_tempdir) - _youtubedl_tempdir_path.mkdir(parents=True, exist_ok=True) + mkdir_p(_youtubedl_tempdir_path) (_youtubedl_tempdir_path / '.ignore').touch(exist_ok=True) _paths = _defaults.get('paths', {}) _paths.update({ 'temp': _youtubedl_tempdir, }) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 0114e76d..1b974cdf 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -93,6 +93,14 @@ SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip( SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR ) +# TUBESYNC_RENAME_ALL_SOURCES: True or False +RENAME_ALL_SOURCES_STR = os.getenv('TUBESYNC_RENAME_ALL_SOURCES', 'False').strip().lower() +RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR ) +# TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories +RENAME_SOURCES_STR = os.getenv('TUBESYNC_RENAME_SOURCES', '') +RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None + + VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a888e91d..3c350ab3 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -177,6 +177,10 @@ COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}' +RENAME_ALL_SOURCES = False +RENAME_SOURCES = None + + try: from .local_settings import * except ImportError as e: