Merge pull request #586 from tcely/rename-files-with-source-format-issue-185

Rename files after a source format change

tubesync/sync/models.py

@@ -14,12 +14,14 @@ from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from common.logger import log
 from common.errors import NoFormatException
 from common.utils import clean_filename, clean_emoji
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
-from .utils import seconds_to_timestr, parse_media_format, filter_response
+from .utils import (seconds_to_timestr, parse_media_format, filter_response,
+                    write_text_file, mkdir_p, directory_and_stem, glob_quote)
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -1566,6 +1568,79 @@ class Media(models.Model):
 
         return str(episode_number)
 
+    def rename_files(self):
+        if self.downloaded and self.media_file:
+            old_video_path = Path(self.media_file.path)
+            new_video_path = Path(get_media_file_path(self, None))
+            if old_video_path.exists() and not new_video_path.exists():
+                old_video_path = old_video_path.resolve(strict=True)
+
+                # move video to destination
+                mkdir_p(new_video_path.parent)
+                log.debug(f'{self!s}: {old_video_path!s} => {new_video_path!s}')
+                old_video_path.rename(new_video_path)
+                log.info(f'Renamed video file for: {self!s}')
+
+                # collect the list of files to move
+                # this should not include the video we just moved
+                (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
+                other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*'))
+                log.info(f'Collected {len(other_paths)} other paths for: {self!s}')
+
+                # adopt orphaned files, if possible
+                media_format = str(self.source.media_format)
+                top_dir_path = Path(self.source.directory_path)
+                if '{key}' in media_format:
+                    fuzzy_paths = list(top_dir_path.rglob('*' + glob_quote(str(self.key)) + '*'))
+                    log.info(f'Collected {len(fuzzy_paths)} fuzzy paths for: {self!s}')
+
+                if new_video_path.exists():
+                    new_video_path = new_video_path.resolve(strict=True)
+
+                    # update the media_file in the db
+                    self.media_file.name = str(new_video_path.relative_to(self.media_file.storage.location))
+                    self.save()
+                    log.info(f'Updated "media_file" in the database for: {self!s}')
+
+                    (new_prefix_path, new_stem) = directory_and_stem(new_video_path)
+
+                    # move and change names to match stem
+                    for other_path in other_paths:
+                        old_file_str = other_path.name
+                        new_file_str = new_stem + old_file_str[len(old_stem):]
+                        new_file_path = Path(new_prefix_path / new_file_str)
+                        log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}')
+                        # it should exist, but check anyway 
+                        if other_path.exists():
+                            log.debug(f'{self!s}: {other_path!s} => {new_file_path!s}')
+                            other_path.replace(new_file_path)
+
+                    for fuzzy_path in fuzzy_paths:
+                        (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path)
+                        old_file_str = fuzzy_path.name
+                        new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
+                        new_file_path = Path(new_prefix_path / new_file_str)
+                        log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}')
+                        # it quite possibly was renamed already
+                        if fuzzy_path.exists() and not new_file_path.exists():
+                            log.debug(f'{self!s}: {fuzzy_path!s} => {new_file_path!s}')
+                            fuzzy_path.rename(new_file_path)
+
+                    # The thumbpath inside the .nfo file may have changed
+                    if self.source.write_nfo and self.source.copy_thumbnails:
+                        write_text_file(new_prefix_path / self.nfopath.name, self.nfoxml)
+                        log.info(f'Wrote new ".nfo" file for: {self!s}')
+
+                    # try to remove empty dirs
+                    parent_dir = old_video_path.parent
+                    try:
+                        while parent_dir.is_dir():
+                            parent_dir.rmdir()
+                            log.info(f'Removed empty directory: {parent_dir!s}')
+                            parent_dir = parent_dir.parent
+                    except OSError as e:
+                        pass
+
 
 class MediaServer(models.Model):
     '''

tubesync/sync/signals.py

@@ -12,7 +12,8 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta
                     download_media_thumbnail, download_media_metadata,
                     map_task_to_instance, check_source_directory_exists,
                     download_media, rescan_media_server, download_source_images,
-                    save_all_media_for_source, get_media_metadata_task)
+                    save_all_media_for_source, rename_all_media_for_source,
+                    get_media_metadata_task)
 from .utils import delete_file
 from .filtering import filter_media
 
@@ -53,7 +54,7 @@ def source_post_save(sender, instance, created, **kwargs):
         if instance.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST and instance.copy_channel_images:
             download_source_images(
                 str(instance.pk),
-                priority=0,
+                priority=2,
                 verbose_name=verbose_name.format(instance.name)
             )
         if instance.index_schedule > 0:
@@ -68,10 +69,28 @@ def source_post_save(sender, instance, created, **kwargs):
                 verbose_name=verbose_name.format(instance.name),
                 remove_existing_tasks=True
             )
+    # Check settings before any rename tasks are scheduled
+    rename_sources_setting = settings.RENAME_SOURCES or list()
+    create_rename_task = (
+        (
+            instance.directory and
+            instance.directory in rename_sources_setting
+        ) or
+        settings.RENAME_ALL_SOURCES
+    )
+    if create_rename_task:
+        verbose_name = _('Renaming all media for source "{}"')
+        rename_all_media_for_source(
+            str(instance.pk),
+            queue=str(instance.pk),
+            priority=1,
+            verbose_name=verbose_name.format(instance.name),
+            remove_existing_tasks=False
+        )
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
-        priority=0,
+        priority=2,
         verbose_name=verbose_name.format(instance.name),
         remove_existing_tasks=True
     )

tubesync/sync/tasks.py

@@ -51,6 +51,7 @@ def map_task_to_instance(task):
         'sync.tasks.download_media': Media,
         'sync.tasks.download_media_metadata': Media,
         'sync.tasks.save_all_media_for_source': Source,
+        'sync.tasks.rename_all_media_for_source': Source,
     }
     MODEL_URL_MAP = {
         Source: 'sync:source',
@@ -516,3 +517,18 @@ def save_all_media_for_source(source_id):
     # flags may need to be recalculated
     for media in Media.objects.filter(source=source):
         media.save()
+
+
+@background(schedule=0)
+def rename_all_media_for_source(source_id):
+    try:
+        source = Source.objects.get(pk=source_id)
+    except Source.DoesNotExist:
+        # Task triggered but the source no longer exists, do nothing
+        log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no '
+                  f'source exists with ID: {source_id}')
+        return
+    for media in Media.objects.filter(source=source):
+        media.rename_files()
+
+

tubesync/sync/utils.py

@@ -94,6 +94,20 @@ def resize_image_to_height(image, width, height):
     return image
 
 
+def glob_quote(filestr):
+    _glob_specials = {
+        '?': '[?]',
+        '*': '[*]',
+        '[': '[[]',
+        ']': '[]]', # probably not needed, but it won't hurt
+    }
+
+    if not isinstance(filestr, str):
+        raise TypeError(f'filestr must be a str, got "{type(filestr)}"')
+
+    return filestr.translate(str.maketrans(_glob_specials))
+
+
 def file_is_editable(filepath):
     '''
         Checks that a file exists and the file is in an allowed predefined tuple of
@@ -115,6 +129,23 @@ def file_is_editable(filepath):
     return False
 
 
+def directory_and_stem(arg_path):
+    filepath = Path(arg_path)
+    stem = Path(filepath.stem)
+    while stem.suffixes and '' != stem.suffix:
+        stem = Path(stem.stem)
+    stem = str(stem)
+    return (filepath.parent, stem,)
+
+
+def mkdir_p(arg_path, mode=0o777):
+    '''
+        Reminder: mode only affects the last directory
+    '''
+    dirpath = Path(arg_path)
+    return dirpath.mkdir(mode=mode, parents=True, exist_ok=True)
+
+
 def write_text_file(filepath, filedata):
     if not isinstance(filedata, str):
         raise TypeError(f'filedata must be a str, got "{type(filedata)}"')

tubesync/sync/youtube.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from django.conf import settings
 from copy import copy
 from common.logger import log
+from .utils import mkdir_p
 import yt_dlp
 
 
@@ -21,7 +22,7 @@ _youtubedl_tempdir = getattr(settings, 'YOUTUBE_DL_TEMPDIR', None)
 if _youtubedl_tempdir:
     _youtubedl_tempdir = str(_youtubedl_tempdir)
     _youtubedl_tempdir_path = Path(_youtubedl_tempdir)
-    _youtubedl_tempdir_path.mkdir(parents=True, exist_ok=True)
+    mkdir_p(_youtubedl_tempdir_path)
     (_youtubedl_tempdir_path / '.ignore').touch(exist_ok=True)
     _paths = _defaults.get('paths', {})
     _paths.update({ 'temp': _youtubedl_tempdir, })

tubesync/tubesync/local_settings.py.container

@@ -93,6 +93,14 @@ SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip(
 SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR )
 
 
+# TUBESYNC_RENAME_ALL_SOURCES: True or False
+RENAME_ALL_SOURCES_STR = os.getenv('TUBESYNC_RENAME_ALL_SOURCES', 'False').strip().lower()
+RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR )
+# TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories
+RENAME_SOURCES_STR = os.getenv('TUBESYNC_RENAME_SOURCES', '')
+RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None
+
+
 VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))
 
 

tubesync/tubesync/settings.py

@@ -177,6 +177,10 @@ COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
 
 
+RENAME_ALL_SOURCES = False
+RENAME_SOURCES = None
+
+
 try:
     from .local_settings import *
 except ImportError as e: