From 3e5711f0f247eb77ecddd19e5131c6010cca7e9e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 11 Dec 2024 08:19:34 -0500
Subject: [PATCH 001/143] Report db.sqlite3 size on dashboard

---
 tubesync/sync/views.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 94e91432..e998d559 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -85,6 +85,12 @@ class DashboardView(TemplateView):
         data['config_dir'] = str(settings.CONFIG_BASE_DIR)
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
         data['database_connection'] = settings.DATABASE_CONNECTION_STR
+        # Add the database filesize when using db.sqlite3
+        db_name = str(settings.DATABASES["default"]["NAME"])
+        db_path = pathlib.Path(db_name) if '/' == db_name[0] else None
+        if db_path and settings.DATABASE_CONNECTION_STR.startswith('sqlite at '):
+            db_size = db_path.stat().st_size
+            data['database_connection'] += f' ({db_size:,} bytes)'
         return data
 
 

From 730275746d78f7ab3e75a65fb7f020a01f263280 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 12 Dec 2024 15:19:54 -0500
Subject: [PATCH 002/143] Use django.db.connection as suggested

---
 tubesync/sync/views.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index e998d559..3fb23044 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -14,7 +14,7 @@ from django.views.generic.detail import SingleObjectMixin
 from django.core.exceptions import SuspiciousFileOperation
 from django.http import HttpResponse
 from django.urls import reverse_lazy
-from django.db import IntegrityError
+from django.db import connection, IntegrityError
 from django.db.models import Q, Count, Sum, When, Case
 from django.forms import Form, ValidationError
 from django.utils.text import slugify
@@ -86,9 +86,9 @@ class DashboardView(TemplateView):
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
         data['database_connection'] = settings.DATABASE_CONNECTION_STR
         # Add the database filesize when using db.sqlite3
-        db_name = str(settings.DATABASES["default"]["NAME"])
+        db_name = str(connection.get_connection_params()['database'])
         db_path = pathlib.Path(db_name) if '/' == db_name[0] else None
-        if db_path and settings.DATABASE_CONNECTION_STR.startswith('sqlite at '):
+        if db_path and 'sqlite' == connection.vendor:
             db_size = db_path.stat().st_size
             data['database_connection'] += f' ({db_size:,} bytes)'
         return data

From 11d97119072bf7a16ed5805cb1057dd061058555 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:46:03 -0500
Subject: [PATCH 003/143] Add rename_all_media_for_source function

---
 tubesync/sync/tasks.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 37983932..aecc86d5 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -486,3 +486,18 @@ def save_all_media_for_source(source_id):
     # flags may need to be recalculated
     for media in Media.objects.filter(source=source):
         media.save()
+
+
+@background(schedule=0)
+def rename_all_media_for_source(source_id):
+    try:
+        source = Source.objects.get(pk=source_id)
+    except Source.DoesNotExist:
+        # Task triggered but the source no longer exists, do nothing
+        log.error(f'Task save_all_media_for_source(pk={source_id}) called but no '
+                  f'source exists with ID: {source_id}')
+        return
+    for media in Media.objects.filter(source=source):
+        media.rename_files()
+
+

From 10c5a989c01081d469fbc847e06c567a7acd7a94 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:22:13 -0500
Subject: [PATCH 004/143] Adjust to accurate function in logged error

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index aecc86d5..635d9c03 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -494,7 +494,7 @@ def rename_all_media_for_source(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the source no longer exists, do nothing
-        log.error(f'Task save_all_media_for_source(pk={source_id}) called but no '
+        log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
         return
     for media in Media.objects.filter(source=source):

From 218c3e6dffe5a58891e8731d8d10f5a1b1a6a422 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 17:39:02 -0500
Subject: [PATCH 005/143] Add Media.rename_files

---
 tubesync/sync/models.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 455a38a5..2fecbc73 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1515,6 +1515,38 @@ class Media(models.Model):
                 return position_counter
             position_counter += 1
 
+    def rename_files(self):
+        if self.downloaded and self.media_file:
+            old_video_path = Path(self.media_file.path)
+            new_video_path = Path(get_media_file_path(self, None))
+            if old_video_path.exists() and not new_video_path.exists():
+                old_video_path = old_video_path.resolve(strict=True)
+                # build the glob to match other files
+                stem = old_video_path
+                while '' != stem.suffix:
+                    stem = Path(stem.stem)
+                old_stem = stem
+                old_prefix_path = old_video_path.parent
+                other_paths = list(old_prefix_path.glob(str(old_stem) + '*'))
+
+                old_video_path.rename(new_video_path)
+                if new_video_path.exists():
+                    new_video_path = new_video_path.resolve(strict=True)
+                    stem = new_video_path
+                    while '' != stem:
+                        stem = Path(stem.stem)
+                    new_stem = stem
+                    new_prefix_path = new_video_path.parent
+                    for other_path in other_paths:
+                        old_file_str = other_path.name
+                        new_file_str = str(new_stem) + old_file_str[len(old_stem):]
+                        new_file_path = Path(new_prefix_path / new_file_str)
+                        other_path.replace(new_file_path)
+
+                    # update the media_file in the db
+                    self.media_file.name = new_video_path.relative_to(media_file_storage.location)
+                    self.save(update_fields={'media_file'})
+
 
 class MediaServer(models.Model):
     '''

From daaebc9c6336d4d8b344cb8d717949805b2fb9b9 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 18:13:47 -0500
Subject: [PATCH 006/143] Add 'rename_all_media_for_source' to TASK_MAP

---
 tubesync/sync/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 635d9c03..7952f8d0 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -50,6 +50,7 @@ def map_task_to_instance(task):
         'sync.tasks.download_media_thumbnail': Media,
         'sync.tasks.download_media': Media,
         'sync.tasks.save_all_media_for_source': Source,
+        'sync.tasks.rename_all_media_for_source': Source,
     }
     MODEL_URL_MAP = {
         Source: 'sync:source',

From b8ec33bba0d201f1f61b76209e94fed2ecf245fd Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 18:24:13 -0500
Subject: [PATCH 007/143] Hook up renaming to source_post_save

---
 tubesync/sync/signals.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 9c541e0a..c61cca87 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -68,6 +68,13 @@ def source_post_save(sender, instance, created, **kwargs):
                 verbose_name=verbose_name.format(instance.name),
                 remove_existing_tasks=True
             )
+    verbose_name = _('Renaming all media for source "{}"')
+    rename_all_media_for_source(
+        str(instance.pk),
+        priority=10,
+        verbose_name=verbose_name.format(instance.name),
+        remove_existing_tasks=True
+    )
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),

From ba0449bd0cd01e5a58dae2d133c5c8accfc30f88 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 18:36:20 -0500
Subject: [PATCH 008/143] Import rename_all_media_for_source too

---
 tubesync/sync/signals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index c61cca87..95f04710 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -12,7 +12,7 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta
                     download_media_thumbnail, download_media_metadata,
                     map_task_to_instance, check_source_directory_exists,
                     download_media, rescan_media_server, download_source_images,
-                    save_all_media_for_source)
+                    save_all_media_for_source, rename_all_media_for_source)
 from .utils import delete_file
 from .filtering import filter_media
 

From 4193e79d0cac3cc78110a71b0546673276e1f21a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 19:29:20 -0500
Subject: [PATCH 009/143] Ensure the directory exists

---
 tubesync/sync/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 2fecbc73..5eb41ac3 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1529,6 +1529,7 @@ class Media(models.Model):
                 old_prefix_path = old_video_path.parent
                 other_paths = list(old_prefix_path.glob(str(old_stem) + '*'))
 
+                new_video_path.parent.mkdir(parents=True, exist_ok=True)
                 old_video_path.rename(new_video_path)
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)

From fdbfd33cc7e462ace8dba4b5a900833da4a50a67 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 22:27:37 -0500
Subject: [PATCH 010/143] Fixes from testing

I don't know how I missed `.suffix` on the later `stem` loop. Oops.
---
 tubesync/sync/models.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 5eb41ac3..0afe44d9 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1523,7 +1523,7 @@ class Media(models.Model):
                 old_video_path = old_video_path.resolve(strict=True)
                 # build the glob to match other files
                 stem = old_video_path
-                while '' != stem.suffix:
+                while stem.suffixes and '' != stem.suffix:
                     stem = Path(stem.stem)
                 old_stem = stem
                 old_prefix_path = old_video_path.parent
@@ -1534,7 +1534,7 @@ class Media(models.Model):
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)
                     stem = new_video_path
-                    while '' != stem:
+                    while stem.suffixes and '' != stem.suffix:
                         stem = Path(stem.stem)
                     new_stem = stem
                     new_prefix_path = new_video_path.parent
@@ -1545,7 +1545,7 @@ class Media(models.Model):
                         other_path.replace(new_file_path)
 
                     # update the media_file in the db
-                    self.media_file.name = new_video_path.relative_to(media_file_storage.location)
+                    self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
                     self.save(update_fields={'media_file'})
 
 

From 278a92b7e1178cf16c4f6fdf7b83dbd63ed26d97 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 17 Dec 2024 22:50:04 -0500
Subject: [PATCH 011/143] Prioritize rename before checking for new media

---
 tubesync/sync/signals.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 95f04710..2bb4e84c 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -71,14 +71,14 @@ def source_post_save(sender, instance, created, **kwargs):
     verbose_name = _('Renaming all media for source "{}"')
     rename_all_media_for_source(
         str(instance.pk),
-        priority=10,
+        priority=0,
         verbose_name=verbose_name.format(instance.name),
         remove_existing_tasks=True
     )
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
-        priority=0,
+        priority=1,
         verbose_name=verbose_name.format(instance.name),
         remove_existing_tasks=True
     )

From 8f0825b57fb0e391abc5e3470750966fc52f09d7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 01:08:14 -0500
Subject: [PATCH 012/143] Rewrite the .nfo file when the thumb path changed

---
 tubesync/sync/models.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 0afe44d9..583ba1c8 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -19,7 +19,7 @@ from common.utils import clean_filename, clean_emoji
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
-from .utils import seconds_to_timestr, parse_media_format
+from .utils import seconds_to_timestr, parse_media_format, write_text_file
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -1548,6 +1548,12 @@ class Media(models.Model):
                     self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
                     self.save(update_fields={'media_file'})
 
+                    # The thumbpath inside the .nfo file may have changed
+                    if self.source.write_nfo and self.source.copy_thumbnails:
+                        nfo_path_tmp = Path(str(self.nfopath) + '.tmp')
+                        write_text_file(nfo_path_tmp, self.nfoxml)
+                        nfo_path_tmp.replace(self.nfopath)
+
 
 class MediaServer(models.Model):
     '''

From 40aed6fc5b5ccbfef47194c89a9b58e45c0bddc0 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 02:58:33 -0500
Subject: [PATCH 013/143] Escape fnmatch special characters in old_stem

---
 tubesync/sync/models.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 583ba1c8..9f6365f5 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -751,6 +751,14 @@ class Media(models.Model):
         STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
         STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
     }
+    # Path.glob uses fnmatch, so we must escape certain characters
+    _glob_specials = {
+        '?': '[?]',
+        '*': '[*]',
+        '[': '[[]',
+        ']': '[]]', # probably not needed, but it won't hurt
+    }
+    _glob_translation = str.maketrans(_glob_specials)
 
     uuid = models.UUIDField(
         _('uuid'),
@@ -1527,7 +1535,8 @@ class Media(models.Model):
                     stem = Path(stem.stem)
                 old_stem = stem
                 old_prefix_path = old_video_path.parent
-                other_paths = list(old_prefix_path.glob(str(old_stem) + '*'))
+                glob_prefix = str(old_stem).translate(_glob_translation)
+                other_paths = list(old_prefix_path.glob(glob_prefix + '*'))
 
                 new_video_path.parent.mkdir(parents=True, exist_ok=True)
                 old_video_path.rename(new_video_path)

From 4b17c08fa58a57416ca98ab87e1d260922db93c5 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 03:18:17 -0500
Subject: [PATCH 014/143] Fixup use of _glob_translation

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 9f6365f5..d73d276f 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1535,7 +1535,7 @@ class Media(models.Model):
                     stem = Path(stem.stem)
                 old_stem = stem
                 old_prefix_path = old_video_path.parent
-                glob_prefix = str(old_stem).translate(_glob_translation)
+                glob_prefix = str(old_stem).translate(self._glob_translation)
                 other_paths = list(old_prefix_path.glob(glob_prefix + '*'))
 
                 new_video_path.parent.mkdir(parents=True, exist_ok=True)

From 9be9b0c58a7cb810897cd300a26c9373d4efec43 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 03:52:51 -0500
Subject: [PATCH 015/143] More fixes after glob worked

stem should be:
1) a stem
2) a string

Replace temporary .nfo into the new directory.
---
 tubesync/sync/models.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index d73d276f..8dbf4102 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1530,26 +1530,26 @@ class Media(models.Model):
             if old_video_path.exists() and not new_video_path.exists():
                 old_video_path = old_video_path.resolve(strict=True)
                 # build the glob to match other files
-                stem = old_video_path
+                stem = Path(old_video_path.stem)
                 while stem.suffixes and '' != stem.suffix:
                     stem = Path(stem.stem)
-                old_stem = stem
+                old_stem = str(stem)
                 old_prefix_path = old_video_path.parent
-                glob_prefix = str(old_stem).translate(self._glob_translation)
+                glob_prefix = old_stem.translate(self._glob_translation)
                 other_paths = list(old_prefix_path.glob(glob_prefix + '*'))
 
                 new_video_path.parent.mkdir(parents=True, exist_ok=True)
                 old_video_path.rename(new_video_path)
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)
-                    stem = new_video_path
+                    stem = Path(new_video_path.stem)
                     while stem.suffixes and '' != stem.suffix:
                         stem = Path(stem.stem)
-                    new_stem = stem
+                    new_stem = str(stem)
                     new_prefix_path = new_video_path.parent
                     for other_path in other_paths:
                         old_file_str = other_path.name
-                        new_file_str = str(new_stem) + old_file_str[len(old_stem):]
+                        new_file_str = new_stem + old_file_str[len(old_stem):]
                         new_file_path = Path(new_prefix_path / new_file_str)
                         other_path.replace(new_file_path)
 
@@ -1561,7 +1561,7 @@ class Media(models.Model):
                     if self.source.write_nfo and self.source.copy_thumbnails:
                         nfo_path_tmp = Path(str(self.nfopath) + '.tmp')
                         write_text_file(nfo_path_tmp, self.nfoxml)
-                        nfo_path_tmp.replace(self.nfopath)
+                        nfo_path_tmp.replace(new_prefix_path / self.nfopath.name)
 
 
 class MediaServer(models.Model):

From 2ffeb5e8ea9a06a7706c90a775193228d6856613 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 04:58:47 -0500
Subject: [PATCH 016/143] Avoid glob matching the video itself

---
 tubesync/sync/models.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 8dbf4102..f0aff808 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1529,6 +1529,10 @@ class Media(models.Model):
             new_video_path = Path(get_media_file_path(self, None))
             if old_video_path.exists() and not new_video_path.exists():
                 old_video_path = old_video_path.resolve(strict=True)
+
+                # mkdir -p destination_dir
+                new_video_path.parent.mkdir(parents=True, exist_ok=True)
+
                 # build the glob to match other files
                 stem = Path(old_video_path.stem)
                 while stem.suffixes and '' != stem.suffix:
@@ -1536,26 +1540,36 @@ class Media(models.Model):
                 old_stem = str(stem)
                 old_prefix_path = old_video_path.parent
                 glob_prefix = old_stem.translate(self._glob_translation)
+
+                # move video to destination
+                old_video_path.rename(new_video_path)
+
+                # collect the list of files to move
+                # this should not include the video we just moved
                 other_paths = list(old_prefix_path.glob(glob_prefix + '*'))
 
-                new_video_path.parent.mkdir(parents=True, exist_ok=True)
-                old_video_path.rename(new_video_path)
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)
+
+                    # update the media_file in the db
+                    self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
+                    self.save(update_fields={'media_file'})
+
+                    # set up new stem and destination directory
                     stem = Path(new_video_path.stem)
                     while stem.suffixes and '' != stem.suffix:
                         stem = Path(stem.stem)
                     new_stem = str(stem)
                     new_prefix_path = new_video_path.parent
+
+                    # move and change names to match stem
                     for other_path in other_paths:
                         old_file_str = other_path.name
                         new_file_str = new_stem + old_file_str[len(old_stem):]
                         new_file_path = Path(new_prefix_path / new_file_str)
-                        other_path.replace(new_file_path)
-
-                    # update the media_file in the db
-                    self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
-                    self.save(update_fields={'media_file'})
+                        # it should exist, but check anyway 
+                        if other_path.exists():
+                            other_path.replace(new_file_path)
 
                     # The thumbpath inside the .nfo file may have changed
                     if self.source.write_nfo and self.source.copy_thumbnails:

From 3a2157e9b37c5bdfc6f2118190be104d5d32e702 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 18 Dec 2024 05:11:26 -0500
Subject: [PATCH 017/143] Use the database values in the current instance

---
 tubesync/sync/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index f0aff808..babbebb6 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1554,6 +1554,7 @@ class Media(models.Model):
                     # update the media_file in the db
                     self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
                     self.save(update_fields={'media_file'})
+                    self.refresh_from_db(fields={'media_file'})
 
                     # set up new stem and destination directory
                     stem = Path(new_video_path.stem)

From a33620f55568aa3b5f29cce352b67da709de0a2b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:09:10 -0500
Subject: [PATCH 018/143] Add directory_and_stem and mkdir_p

---
 tubesync/sync/utils.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 1c00b4a2..ae81cc69 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -112,6 +112,23 @@ def file_is_editable(filepath):
     return False
 
 
+def directory_and_stem(arg_path):
+    filepath = Path(arg_path)
+    stem = Path(filepath.stem)
+    while stem.suffixes and '' != stem.suffix:
+        stem = Path(stem.stem)
+    stem = str(stem)
+    return (filepath.parent, stem, filepath.suffixes,)
+
+
+def mkdir_p(arg_path, mode=0o777):
+    '''
+        Reminder: mode only affects the last directory
+    '''
+    dirpath = Path(arg_path)
+    return dirpath.mkdir(mode=mode, parents=True, exist_ok=True)
+
+
 def write_text_file(filepath, filedata):
     if not isinstance(filedata, str):
         raise ValueError(f'filedata must be a str, got "{type(filedata)}"')

From 80a7718a64d925e3416abb62a4f86f780cb3ce1b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:30:43 -0500
Subject: [PATCH 019/143] Use mkdir_p in youtube.py

---
 tubesync/sync/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index d51c7b45..9cec26d1 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -9,6 +9,7 @@ from pathlib import Path
 from django.conf import settings
 from copy import copy
 from common.logger import log
+from .utils import mkdir_p
 import yt_dlp
 
 
@@ -21,7 +22,7 @@ _youtubedl_tempdir = getattr(settings, 'YOUTUBE_DL_TEMPDIR', None)
 if _youtubedl_tempdir:
     _youtubedl_tempdir = str(_youtubedl_tempdir)
     _youtubedl_tempdir_path = Path(_youtubedl_tempdir)
-    _youtubedl_tempdir_path.mkdir(parents=True, exist_ok=True)
+    mkdir_p(_youtubedl_tempdir_path)
     (_youtubedl_tempdir_path / '.ignore').touch(exist_ok=True)
     _paths = _defaults.get('paths', {})
     _paths.update({ 'temp': _youtubedl_tempdir, })

From 02f1d0857089a79e7d94eec2ef68788000b927d8 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:36:15 -0500
Subject: [PATCH 020/143] Use mkdir_p in models.py

---
 tubesync/sync/models.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index babbebb6..d73ac11e 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -19,7 +19,8 @@ from common.utils import clean_filename, clean_emoji
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
-from .utils import seconds_to_timestr, parse_media_format, write_text_file
+from .utils import (seconds_to_timestr, parse_media_format, write_text_file,
+                    mkdir_p)
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -1530,8 +1531,7 @@ class Media(models.Model):
             if old_video_path.exists() and not new_video_path.exists():
                 old_video_path = old_video_path.resolve(strict=True)
 
-                # mkdir -p destination_dir
-                new_video_path.parent.mkdir(parents=True, exist_ok=True)
+                mkdir_p(new_video_path.parent)
 
                 # build the glob to match other files
                 stem = Path(old_video_path.stem)

From 0e18d6d2bf78cef95daebbe185a87c2b7794d100 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:46:11 -0500
Subject: [PATCH 021/143] Don't return suffixes

This isn't useful.
The caller can create a `Path` to get this instead.
---
 tubesync/sync/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index ae81cc69..9d4ca4b4 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -118,7 +118,7 @@ def directory_and_stem(arg_path):
     while stem.suffixes and '' != stem.suffix:
         stem = Path(stem.stem)
     stem = str(stem)
-    return (filepath.parent, stem, filepath.suffixes,)
+    return (filepath.parent, stem,)
 
 
 def mkdir_p(arg_path, mode=0o777):

From 35912f0fe04654d02db9117a3eece33f31430de7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:56:31 -0500
Subject: [PATCH 022/143] Use directory_and_stem in models.py

---
 tubesync/sync/models.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index d73ac11e..f3e842c4 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -20,7 +20,7 @@ from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
 from .utils import (seconds_to_timestr, parse_media_format, write_text_file,
-                    mkdir_p)
+                    mkdir_p, directory_and_stem)
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -1534,11 +1534,7 @@ class Media(models.Model):
                 mkdir_p(new_video_path.parent)
 
                 # build the glob to match other files
-                stem = Path(old_video_path.stem)
-                while stem.suffixes and '' != stem.suffix:
-                    stem = Path(stem.stem)
-                old_stem = str(stem)
-                old_prefix_path = old_video_path.parent
+                (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
                 glob_prefix = old_stem.translate(self._glob_translation)
 
                 # move video to destination
@@ -1556,12 +1552,7 @@ class Media(models.Model):
                     self.save(update_fields={'media_file'})
                     self.refresh_from_db(fields={'media_file'})
 
-                    # set up new stem and destination directory
-                    stem = Path(new_video_path.stem)
-                    while stem.suffixes and '' != stem.suffix:
-                        stem = Path(stem.stem)
-                    new_stem = str(stem)
-                    new_prefix_path = new_video_path.parent
+                    (new_prefix_path, new_stem) = directory_and_stem(new_video_path)
 
                     # move and change names to match stem
                     for other_path in other_paths:

From 9168e82e8e3e8c7c930442e50f2909f691bec73c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 17:20:06 -0500
Subject: [PATCH 023/143] Add glob_quote

---
 tubesync/sync/utils.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 9d4ca4b4..dc4899c5 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -92,6 +92,20 @@ def resize_image_to_height(image, width, height):
     return image
 
 
+def glob_quote(filestr):
+    _glob_specials = {
+        '?': '[?]',
+        '*': '[*]',
+        '[': '[[]',
+        ']': '[]]', # probably not needed, but it won't hurt
+    }
+
+    if not isinstance(filestr, str):
+        raise ValueError(f'filestr must be a str, got "{type(filestr)}"')
+
+    return filestr.translate(str.maketrans(_glob_specials))
+
+
 def file_is_editable(filepath):
     '''
         Checks that a file exists and the file is in an allowed predefined tuple of

From e3bcd783384858e953bb725a8c7ad5761d918eb9 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 17:28:15 -0500
Subject: [PATCH 024/143] Use glob_quote in models.py

---
 tubesync/sync/models.py | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index f3e842c4..6b5bd8f7 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -20,7 +20,7 @@ from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
 from .utils import (seconds_to_timestr, parse_media_format, write_text_file,
-                    mkdir_p, directory_and_stem)
+                    mkdir_p, directory_and_stem, glob_quote)
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -752,14 +752,6 @@ class Media(models.Model):
         STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
         STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
     }
-    # Path.glob uses fnmatch, so we must escape certain characters
-    _glob_specials = {
-        '?': '[?]',
-        '*': '[*]',
-        '[': '[[]',
-        ']': '[]]', # probably not needed, but it won't hurt
-    }
-    _glob_translation = str.maketrans(_glob_specials)
 
     uuid = models.UUIDField(
         _('uuid'),
@@ -1531,18 +1523,14 @@ class Media(models.Model):
             if old_video_path.exists() and not new_video_path.exists():
                 old_video_path = old_video_path.resolve(strict=True)
 
-                mkdir_p(new_video_path.parent)
-
-                # build the glob to match other files
-                (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
-                glob_prefix = old_stem.translate(self._glob_translation)
-
                 # move video to destination
+                mkdir_p(new_video_path.parent)
                 old_video_path.rename(new_video_path)
 
                 # collect the list of files to move
                 # this should not include the video we just moved
-                other_paths = list(old_prefix_path.glob(glob_prefix + '*'))
+                (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
+                other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*'))
 
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)

From 521b903faa851b646e5a3ec8e3fd58f083a494af Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 18:01:30 -0500
Subject: [PATCH 025/143] Try to rename lost files by key

---
 tubesync/sync/models.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 6b5bd8f7..26639d2d 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1532,6 +1532,11 @@ class Media(models.Model):
                 (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
                 other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*'))
 
+                # adopt orphaned files, if possible
+                media_format = str(self.source.media_format)
+                if '{key}' in media_format:
+                    fuzzy_paths = list(old_prefix_path.glob('*' + glob_quote(str(self.key)) + '*'))
+
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)
 
@@ -1551,12 +1556,27 @@ class Media(models.Model):
                         if other_path.exists():
                             other_path.replace(new_file_path)
 
+                    for fuzzy_path in fuzzy_paths:
+                        (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path)
+                        old_file_str = fuzzy_path.name
+                        new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
+                        new_file_path = Path(new_prefix_path / new_file_str) 
+                        # it quite possibly was renamed already
+                        if fuzzy_path.exists() and not new_file_path.exists():
+                            fuzzy_path.rename(new_file_path)
+
                     # The thumbpath inside the .nfo file may have changed
                     if self.source.write_nfo and self.source.copy_thumbnails:
                         nfo_path_tmp = Path(str(self.nfopath) + '.tmp')
                         write_text_file(nfo_path_tmp, self.nfoxml)
                         nfo_path_tmp.replace(new_prefix_path / self.nfopath.name)
 
+                    # try to remove empty dirs
+                    try:
+                        old_video_path.parent.rmdir()
+                    except:
+                        pass
+
 
 class MediaServer(models.Model):
     '''

From 344759e5227b445c27da21ebcffbb8c3301cddf7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 18:18:23 -0500
Subject: [PATCH 026/143] Try not to leave empty directories around

---
 tubesync/sync/models.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 26639d2d..94901e8f 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1572,9 +1572,12 @@ class Media(models.Model):
                         nfo_path_tmp.replace(new_prefix_path / self.nfopath.name)
 
                     # try to remove empty dirs
+                    parent_dir = old_video_path.parent
                     try:
-                        old_video_path.parent.rmdir()
-                    except:
+                        while parent_dir.is_dir():
+                            parent_dir.rmdir()
+                            parent_dir = parent_dir.parent
+                    except OSError as e:
                         pass
 
 

From 72e4095354f7e9bf2199a808d8a39e45dbc2bb42 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 21:26:10 -0500
Subject: [PATCH 027/143] Use TypeError in glob_quote

---
 tubesync/sync/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index dc4899c5..4dd85126 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -101,7 +101,7 @@ def glob_quote(filestr):
     }
 
     if not isinstance(filestr, str):
-        raise ValueError(f'filestr must be a str, got "{type(filestr)}"')
+        raise TypeError(f'filestr must be a str, got "{type(filestr)}"')
 
     return filestr.translate(str.maketrans(_glob_specials))
 

From 4f56ebd1cee3c41119ab42f3dc72c81484fcc5ad Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 19 Dec 2024 21:34:28 -0500
Subject: [PATCH 028/143] Depend on the function to write a temporary file

---
 tubesync/sync/models.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 94901e8f..ca4e79ec 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1567,9 +1567,7 @@ class Media(models.Model):
 
                     # The thumbpath inside the .nfo file may have changed
                     if self.source.write_nfo and self.source.copy_thumbnails:
-                        nfo_path_tmp = Path(str(self.nfopath) + '.tmp')
-                        write_text_file(nfo_path_tmp, self.nfoxml)
-                        nfo_path_tmp.replace(new_prefix_path / self.nfopath.name)
+                        write_text_file(new_prefix_path / self.nfopath.name, self.nfoxml)
 
                     # try to remove empty dirs
                     parent_dir = old_video_path.parent

From 2ee635118bfcabe5685bc0fda25f9d8c54ee2884 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 20 Dec 2024 02:56:31 -0500
Subject: [PATCH 029/143] Added logging to rename_files

Also, the fuzzy matching, by key in the filename, begins at the source directory.
---
 tubesync/sync/models.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index ca4e79ec..3d25e427 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -14,6 +14,7 @@ from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from common.logger import log
 from common.errors import NoFormatException
 from common.utils import clean_filename, clean_emoji
 from .youtube import (get_media_info as get_youtube_media_info,
@@ -1525,17 +1526,22 @@ class Media(models.Model):
 
                 # move video to destination
                 mkdir_p(new_video_path.parent)
+                log.debug(f'{self!s}: {old_video_path!s} => {new_video_path!s}')
                 old_video_path.rename(new_video_path)
+                log.info(f'Renamed video file for: {self!s}')
 
                 # collect the list of files to move
                 # this should not include the video we just moved
                 (old_prefix_path, old_stem) = directory_and_stem(old_video_path)
                 other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*'))
+                log.info(f'Collected {len(other_paths)} other paths for: {self!s}')
 
                 # adopt orphaned files, if possible
                 media_format = str(self.source.media_format)
+                top_dir_path = Path(self.source.directory_path)
                 if '{key}' in media_format:
-                    fuzzy_paths = list(old_prefix_path.glob('*' + glob_quote(str(self.key)) + '*'))
+                    fuzzy_paths = list(top_dir_path.rglob('*' + glob_quote(str(self.key)) + '*'))
+                    log.info(f'Collected {len(fuzzy_paths)} fuzzy paths for: {self!s}')
 
                 if new_video_path.exists():
                     new_video_path = new_video_path.resolve(strict=True)
@@ -1544,6 +1550,7 @@ class Media(models.Model):
                     self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
                     self.save(update_fields={'media_file'})
                     self.refresh_from_db(fields={'media_file'})
+                    log.info(f'Updated "media_file" in the database for: {self!s}')
 
                     (new_prefix_path, new_stem) = directory_and_stem(new_video_path)
 
@@ -1552,28 +1559,34 @@ class Media(models.Model):
                         old_file_str = other_path.name
                         new_file_str = new_stem + old_file_str[len(old_stem):]
                         new_file_path = Path(new_prefix_path / new_file_str)
+                        log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}')
                         # it should exist, but check anyway 
                         if other_path.exists():
+                            log.debug(f'{self!s}: {other_path!s} => {new_file_path!s}')
                             other_path.replace(new_file_path)
 
                     for fuzzy_path in fuzzy_paths:
                         (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path)
                         old_file_str = fuzzy_path.name
                         new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
-                        new_file_path = Path(new_prefix_path / new_file_str) 
+                        new_file_path = Path(new_prefix_path / new_file_str)
+                        log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}')
                         # it quite possibly was renamed already
                         if fuzzy_path.exists() and not new_file_path.exists():
+                            log.debug(f'{self!s}: {fuzzy_path!s} => {new_file_path!s}')
                             fuzzy_path.rename(new_file_path)
 
                     # The thumbpath inside the .nfo file may have changed
                     if self.source.write_nfo and self.source.copy_thumbnails:
                         write_text_file(new_prefix_path / self.nfopath.name, self.nfoxml)
+                        log.info(f'Wrote new ".nfo" file for: {self!s}')
 
                     # try to remove empty dirs
                     parent_dir = old_video_path.parent
                     try:
                         while parent_dir.is_dir():
                             parent_dir.rmdir()
+                            log.info(f'Removed empty directory: {parent_dir!s}')
                             parent_dir = parent_dir.parent
                     except OSError as e:
                         pass

From f7b36a47da855badf381100a46be326c51af3f60 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 23 Dec 2024 19:31:38 -0500
Subject: [PATCH 030/143] Use the media_file storage location

This should be the same, but this way works outside of this file too.
---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 3d25e427..ab224036 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1547,7 +1547,7 @@ class Media(models.Model):
                     new_video_path = new_video_path.resolve(strict=True)
 
                     # update the media_file in the db
-                    self.media_file.name = str(new_video_path.relative_to(media_file_storage.location))
+                    self.media_file.name = str(new_video_path.relative_to(self.media_file.storage.location))
                     self.save(update_fields={'media_file'})
                     self.refresh_from_db(fields={'media_file'})
                     log.info(f'Updated "media_file" in the database for: {self!s}')

From 59865a885bf411bfeeed775a32c20113349eeb1b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 2 Jan 2025 02:39:19 -0500
Subject: [PATCH 031/143] Match another variation of the message

```
{key}: This video is available to this channel's members on level: Level 2: Contributor (or any higher level). Join this YouTube channel from your computer or Android app.
```
---
 tubesync/sync/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 1eac4c7f..5fdef3cb 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -81,6 +81,8 @@ def _subscriber_only(msg='', response=None):
             return True
         if ': Join this channel' in msg:
             return True
+        if 'Join this YouTube channel' in msg:
+            return True
     else:
         # ignore msg entirely
         if not isinstance(response, dict):

From e64f71a9704b8d72d2b6b0f693605c6bc7988977 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 6 Jan 2025 10:23:17 -0500
Subject: [PATCH 032/143] Don't chmod a+r when it already has those permissions

---
 tubesync/sync/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 3e29fe3f..73c8f394 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -123,7 +123,8 @@ def write_text_file(filepath, filedata):
         bytes_written = f.write(filedata)
     # chmod a+r temp_file
     old_mode = new_filepath.stat().st_mode
-    new_filepath.chmod(0o444 | old_mode)
+    if 0o444 != (0o444 & old_mode):
+        new_filepath.chmod(0o444 | old_mode)
     if not file_is_editable(new_filepath):
         new_filepath.unlink()
         raise ValueError(f'File cannot be edited or removed: {filepath}')

From 215aa64f2d6b3e5ead44f8a37ff382358eee1ace Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 6 Jan 2025 10:35:37 -0500
Subject: [PATCH 033/143] Change to a logged warning for NFO permission
 problems

---
 tubesync/sync/tasks.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 3df651ba..b36b5d49 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -446,7 +446,11 @@ def download_media(media_id):
         # If selected, write an NFO file
         if media.source.write_nfo:
             log.info(f'Writing media NFO file to: {media.nfopath}')
-            write_text_file(media.nfopath, media.nfoxml)
+            try:
+                write_text_file(media.nfopath, media.nfoxml)
+            except PermissionError as e:
+                log.warn(f'A permissions problem occured when writing the new media NFO file: {e.msg}')
+                pass
         # Schedule a task to update media servers
         for mediaserver in MediaServer.objects.all():
             log.info(f'Scheduling media server updates')

From 8c22b6c99efb464dfb450707440c6162f04b7b46 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 00:05:57 -0500
Subject: [PATCH 034/143] Add response filtering

These functions aren't being used yet, they will be tested against my database before that happens.
---
 tubesync/sync/utils.py | 62 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 3e29fe3f..e44cef1f 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -170,6 +170,68 @@ def normalize_codec(codec_str):
     return result
 
 
+def _url_keys(arg_dict, filter_func):
+    result = {}
+    for key in arg_dict.keys():
+        if 'url' in key:
+            result.update(
+                {key: (key, filter_func(key=key, url=arg_dict[key]),)}
+            )
+    return result
+
+
+def _drop_url_keys(arg_dict, key, filter_func):
+    if key in arg_dict.keys():
+        for val_dict in arg_dict[key]:
+            for url_key in _url_keys(val_dict, filter_func):
+                if url_key[1] is True:
+                    del val_dict[url_key[0]]
+
+
+def filter_response(response_dict):
+    '''
+        Clean up the response so as to not store useless metadata in the database.
+    '''
+    # raise an exception for an unexpected argument type
+    if not isinstance(filedata, dict):
+        raise TypeError(f'filedata must be a dict, got "{type(filedata)}"')
+    # optimize the empty case
+    if not response_dict:
+        return response_dict
+
+    # beginning of formats cleanup {{{
+    # drop urls that expire, or restrict IPs
+    def drop_format_url(**kwargs):
+        url = kwargs['url']
+        return (
+            url
+            and '://' in url
+            and (
+                '/ip/' in url
+                or '/expire/' in url
+            )
+        )
+
+    _drop_url_keys(response_dict, 'formats', drop_format_url)
+    _drop_url_keys(response_dict, 'requested_formats', drop_format_url)
+    # end of formats cleanup }}}
+
+    # beginning of automatic_captions cleanup {{{
+    # drop urls that expire, or restrict IPs
+    def drop_auto_caption_url(**kwargs):
+        url = kwargs['url']
+        return (
+            url
+            and '://' in url
+            and '&expire=' in url
+        )
+
+    _drop_url_keys(response_dict, 'automatic_captions', drop_auto_caption_url)
+    # end of automatic_captions cleanup }}}
+
+    return response_dict
+
+
 def parse_media_format(format_dict):
     '''
         This parser primarily adapts the format dict returned by youtube-dl into a

From 63fa97cc5842af7805c3efb1c8d58971b096893d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 00:43:59 -0500
Subject: [PATCH 035/143] More compact JSON

The software doesn't need an extra space per key.
---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 3df651ba..080dff6d 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -304,7 +304,7 @@ def download_media_metadata(media_id):
         return
     source = media.source
     metadata = media.index_metadata()
-    media.metadata = json.dumps(metadata, default=json_serial)
+    media.metadata = json.dumps(metadata, separators=(',', ':'), default=json_serial)
     upload_date = media.upload_date
     # Media must have a valid upload date
     if upload_date:

From 8c31720bf707b0b12713af0e8a5a356f3bc6255d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 01:33:06 -0500
Subject: [PATCH 036/143] Log the reduction of metadata length

---
 tubesync/sync/models.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 2037492d..7ae68729 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -19,7 +19,7 @@ from common.utils import clean_filename, clean_emoji
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media,
                       get_channel_image_info as get_youtube_channel_image_info)
-from .utils import seconds_to_timestr, parse_media_format
+from .utils import seconds_to_timestr, parse_media_format, filter_response
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -1143,12 +1143,27 @@ class Media(models.Model):
     def has_metadata(self):
         return self.metadata is not None
 
+
+    def reduce_data(self, data):
+        from common.logger import log
+        from common.utils import json_serial
+        # log the results of filtering / compacting on metadata size
+        filtered_data = filter_response(data)
+        compact_metadata = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
+        old_mdl = len(self.metadata)
+        new_mdl = len(compact_metadata)
+        if old_mdl > new_mdl:
+            delta = old_mdl - new_mdl
+            log.info(f'{self.key}: metadata reduced by {delta,} characters ({old_mdl,} -> {new_mdl,})')
+
+
     @property
     def loaded_metadata(self):
         try:
             data = json.loads(self.metadata)
             if not isinstance(data, dict):
                 return {}
+            self.reduce_data(data)
             return data
         except Exception as e:
             return {}

From 25d2ff680270aa9e4188233cba3770cd9dc5275e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 02:12:22 -0500
Subject: [PATCH 037/143] Don't reduce the actual data yet

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 7ae68729..44f24dfb 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1163,7 +1163,7 @@ class Media(models.Model):
             data = json.loads(self.metadata)
             if not isinstance(data, dict):
                 return {}
-            self.reduce_data(data)
+            self.reduce_data(json.loads(self.metadata))
             return data
         except Exception as e:
             return {}

From 2f34fff7133754c05d348d50e43442a481c8adfc Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 02:55:05 -0500
Subject: [PATCH 038/143] Fixes from testing

The `automatic_captions` has a layer for language codes that I didn't account for.

The type checking was copied and I didn't adjust for the arguments in this function.
---
 tubesync/sync/utils.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 162146eb..b85abaab 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -194,8 +194,8 @@ def filter_response(response_dict):
         Clean up the response so as to not store useless metadata in the database.
     '''
     # raise an exception for an unexpected argument type
-    if not isinstance(filedata, dict):
-        raise TypeError(f'filedata must be a dict, got "{type(filedata)}"')
+    if not isinstance(response_dict, dict):
+        raise TypeError(f'response_dict must be a dict, got "{type(response_dict)}"')
     # optimize the empty case
     if not response_dict:
         return response_dict
@@ -227,7 +227,11 @@ def filter_response(response_dict):
             and '&expire=' in url
         )
 
-    _drop_url_keys(response_dict, 'automatic_captions', drop_auto_caption_url)
+    ac_key = 'automatic_captions'
+    if ac_key in response_dict.keys():
+        ac_dict = response_dict[ac_key]
+        for lang_code in ac_dict:
+            _drop_url_keys(ac_dict, lang_code, drop_auto_caption_url)
     # end of automatic_captions cleanup }}}
 
     return response_dict

From 9a4101a0a147f3fe0ee91c13197a077f1f27cd3e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 03:18:39 -0500
Subject: [PATCH 039/143] Fix formatting

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 44f24dfb..077a8283 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1154,7 +1154,7 @@ class Media(models.Model):
         new_mdl = len(compact_metadata)
         if old_mdl > new_mdl:
             delta = old_mdl - new_mdl
-            log.info(f'{self.key}: metadata reduced by {delta,} characters ({old_mdl,} -> {new_mdl,})')
+            log.info(f'{self.key}: metadata reduced by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')
 
 
     @property

From db25fa80294e035b1742fac2e044d2ff7de27464 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 03:35:58 -0500
Subject: [PATCH 040/143] Adjusted comment

---
 tubesync/sync/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index b85abaab..108cd757 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -218,7 +218,7 @@ def filter_response(response_dict):
     # end of formats cleanup }}}
 
     # beginning of automatic_captions cleanup {{{
-    # drop urls that expire, or restrict IPs
+    # drop urls that expire
     def drop_auto_caption_url(**kwargs):
         url = kwargs['url']
         return (

From 431de2e0dfa606d5a725a475159afe5fe370a251 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 04:11:14 -0500
Subject: [PATCH 041/143] Loop over a set of keys for each URL type

---
 tubesync/sync/utils.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 108cd757..f66348b4 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -213,13 +213,13 @@ def filter_response(response_dict):
             )
         )
 
-    _drop_url_keys(response_dict, 'formats', drop_format_url)
-    _drop_url_keys(response_dict, 'requested_formats', drop_format_url)
+    for key in frozenset(('formats', 'requested_formats',)):
+        _drop_url_keys(response_dict, key, drop_format_url)
     # end of formats cleanup }}}
 
-    # beginning of automatic_captions cleanup {{{
+    # beginning of subtitles cleanup {{{
     # drop urls that expire
-    def drop_auto_caption_url(**kwargs):
+    def drop_subtitles_url(**kwargs):
         url = kwargs['url']
         return (
             url
@@ -227,12 +227,13 @@ def filter_response(response_dict):
             and '&expire=' in url
         )
 
-    ac_key = 'automatic_captions'
-    if ac_key in response_dict.keys():
-        ac_dict = response_dict[ac_key]
-        for lang_code in ac_dict:
-            _drop_url_keys(ac_dict, lang_code, drop_auto_caption_url)
-    # end of automatic_captions cleanup }}}
+    # beginning of automatic_captions cleanup {{{
+    for key in frozenset(('subtitles', 'automatic_captions',)):
+        if key in response_dict.keys():
+            key_dict = response_dict[key]
+            for lang_code in key_dict:
+                _drop_url_keys(key_dict, lang_code, drop_subtitles_url)
+    # end of subtitles cleanup }}}
 
     return response_dict
 

From 7b8d11791d9725191146304f612ae7e2f7d3d0ec Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 05:39:50 -0500
Subject: [PATCH 042/143] Drop keys from formats that cannot be useful

---
 tubesync/sync/utils.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index f66348b4..8e98857e 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -213,8 +213,20 @@ def filter_response(response_dict):
             )
         )
 
+    # these format keys are not useful to us
+    drop_keys = frozenset((
+        'downloader_options',
+        'fragments',
+        'http_headers',
+        '__needs_testing',
+        '__working',
+    ))
     for key in frozenset(('formats', 'requested_formats',)):
         _drop_url_keys(response_dict, key, drop_format_url)
+        if key in response_dict.keys():
+            for format in response_dict[key]:
+                for drop_key in drop_keys:
+                    del format[drop_key]
     # end of formats cleanup }}}
 
     # beginning of subtitles cleanup {{{
@@ -227,7 +239,6 @@ def filter_response(response_dict):
             and '&expire=' in url
         )
 
-    # beginning of automatic_captions cleanup {{{
     for key in frozenset(('subtitles', 'automatic_captions',)):
         if key in response_dict.keys():
             key_dict = response_dict[key]

From c7457e94ac1f27c04f912a086b9cc766f4ab5882 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 05:58:50 -0500
Subject: [PATCH 043/143] Check that the drop_key exists

---
 tubesync/sync/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 8e98857e..f73e243b 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -226,7 +226,8 @@ def filter_response(response_dict):
         if key in response_dict.keys():
             for format in response_dict[key]:
                 for drop_key in drop_keys:
-                    del format[drop_key]
+                    if drop_key in format.keys():
+                        del format[drop_key]
     # end of formats cleanup }}}
 
     # beginning of subtitles cleanup {{{

From 2d85bcbe14c0701782d5c76b0cb36116be193d08 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 06:20:01 -0500
Subject: [PATCH 044/143] Use a distinct try to log errors

---
 tubesync/sync/models.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 077a8283..6bcac984 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1159,11 +1159,17 @@ class Media(models.Model):
 
     @property
     def loaded_metadata(self):
+        from common.logger import log
+        try:
+            self.reduce_data(json.loads(self.metadata))
+        except Exception as e:
+            log.error(f'reduce_data: {e.msg}')
+            pass
+
         try:
             data = json.loads(self.metadata)
             if not isinstance(data, dict):
                 return {}
-            self.reduce_data(json.loads(self.metadata))
             return data
         except Exception as e:
             return {}

From 8ac5b36eee9a504d0f0b5a9092c5120fa7f8ecbf Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 06:38:56 -0500
Subject: [PATCH 045/143] Use the exception function for traceback

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 6bcac984..54fcdaa6 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1163,7 +1163,7 @@ class Media(models.Model):
         try:
             self.reduce_data(json.loads(self.metadata))
         except Exception as e:
-            log.error(f'reduce_data: {e.msg}')
+            log.exception('reduce_data: %s', e)
             pass
 
         try:

From 779370122847bb24484181834a299f7e3f41ed1f Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 13:01:06 -0500
Subject: [PATCH 046/143] Simplify results from _url_keys

Also, name the tuple values when using the results.
---
 tubesync/sync/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index f73e243b..170b2a51 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -176,7 +176,7 @@ def _url_keys(arg_dict, filter_func):
     for key in arg_dict.keys():
         if 'url' in key:
             result.update(
-                {key: (key, filter_func(key=key, url=arg_dict[key]),)}
+                {key: (filter_func(key=key, url=arg_dict[key]),)}
             )
     return result
 
@@ -184,9 +184,9 @@ def _url_keys(arg_dict, filter_func):
 def _drop_url_keys(arg_dict, key, filter_func):
     if key in arg_dict.keys():
         for val_dict in arg_dict[key]:
-            for url_key in _url_keys(val_dict, filter_func):
-                if url_key[1] is True:
-                    del val_dict[url_key[0]]
+            for url_key, remove in _url_keys(val_dict, filter_func).items():
+                if remove is True:
+                    del val_dict[url_key]
 
 
 def filter_response(response_dict):

From 1c432ccce127439bc722e4d0727d545794d51e4e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 13:49:58 -0500
Subject: [PATCH 047/143] Some formats are using a different URL

---
 tubesync/sync/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 170b2a51..14e7505f 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -176,7 +176,7 @@ def _url_keys(arg_dict, filter_func):
     for key in arg_dict.keys():
         if 'url' in key:
             result.update(
-                {key: (filter_func(key=key, url=arg_dict[key]),)}
+                {key: filter_func(key=key, url=arg_dict[key])}
             )
     return result
 
@@ -209,7 +209,9 @@ def filter_response(response_dict):
             and '://' in url
             and (
                 '/ip/' in url
+                or 'ip=' in url
                 or '/expire/' in url
+                or 'expire=' in url
             )
         )
 

From 6e116899a72dfe9cf8f9b94442274b276a113715 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 7 Jan 2025 23:45:25 -0500
Subject: [PATCH 048/143] We don't need to keep bash running

---
 config/root/etc/s6-overlay/s6-rc.d/nginx/run | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run
index 6981f2e9..87769e62 100755
--- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run
+++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run
@@ -2,4 +2,4 @@
 
 cd /
 
-/usr/sbin/nginx
+exec /usr/sbin/nginx

From ab5e63f6433898d6545b42cb7f80af92734e3e07 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 8 Jan 2025 00:12:21 -0500
Subject: [PATCH 049/143] Make better use of CPU cache with nginx

---
 config/root/etc/nginx/nginx.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf
index 14c5aea9..f09c02e1 100644
--- a/config/root/etc/nginx/nginx.conf
+++ b/config/root/etc/nginx/nginx.conf
@@ -2,6 +2,7 @@ daemon off;
 
 user app;
 worker_processes auto;
+worker_cpu_affinity auto;
 pid /run/nginx.pid;
 
 events {

From f0b7d31949dbdf0203efb716d0b6a72999582c7b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 8 Jan 2025 00:31:30 -0500
Subject: [PATCH 050/143] Test and log ffmpeg version output earlier

Running the ffmpeg in an earlier (hopefully cached) layer should clean up the logs a bit.

On a related note, shadowing the environment variable was causing some confusing log output, so stop doing that as well.
---
 Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 76bb21b2..c552f9d5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -109,6 +109,7 @@ RUN decide_arch() { \
   _file="/tmp/ffmpeg-${ARCH}.tar.xz" && \
   download_expected_file ffmpeg "${TARGETARCH}" "${_file}" && \
   tar -xvvpf "${_file}" --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" "ffprobe" && rm -f "${_file}" && \
+  /usr/local/bin/ffmpeg -version && \
   file /usr/local/bin/ff* && \
   # Clean up
   apt-get -y autoremove --purge curl file binutils xz-utils && \
@@ -217,10 +218,9 @@ RUN set -x && \
 
 # Append software versions
 RUN set -x && \
-  /usr/local/bin/ffmpeg -version && \
-  FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \
-  test -n "${FFMPEG_VERSION}" && \
-  printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py
+  ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \
+  test -n "${ffmpeg_version}" && \
+  printf -- "ffmpeg_version = '%s'\n" "${ffmpeg_version}" >> /app/common/third_party_versions.py
 
 # Copy root
 COPY config/root /

From 7b42213bbb50c172ffe23dfbb6728913c3b1ebf4 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 8 Jan 2025 00:36:37 -0500
Subject: [PATCH 051/143] Keep curl and add less

These are very useful when using the shell inside the container and don't use much space.
---
 Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c552f9d5..880dd677 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -112,7 +112,7 @@ RUN decide_arch() { \
   /usr/local/bin/ffmpeg -version && \
   file /usr/local/bin/ff* && \
   # Clean up
-  apt-get -y autoremove --purge curl file binutils xz-utils && \
+  apt-get -y autoremove --purge file binutils xz-utils && \
   rm -rf /var/lib/apt/lists/* && \
   rm -rf /var/cache/apt/* && \
   rm -rf /tmp/*
@@ -132,6 +132,8 @@ RUN set -x && \
   python3 \
   python3-wheel \
   redis-server \
+  curl \
+  less \
   && apt-get -y autoclean && \
   rm -rf /var/lib/apt/lists/* && \
   rm -rf /var/cache/apt/* && \

From d35f52f8acb07c30f81c855a855b63d284dbaedf Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 8 Jan 2025 11:31:23 -0500
Subject: [PATCH 052/143] Drop /expire/ URLs from automatic_captions too

---
 tubesync/sync/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 14e7505f..b424528b 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -239,7 +239,10 @@ def filter_response(response_dict):
         return (
             url
             and '://' in url
-            and '&expire=' in url
+            and (
+                '/expire/' in url
+                or '&expire=' in url
+            )
         )
 
     for key in frozenset(('subtitles', 'automatic_captions',)):

From ad10bcfa61af480fd9be9b3f7a97baeba18e033d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 8 Jan 2025 22:48:23 -0500
Subject: [PATCH 053/143] Log both compacted and reduced sizes

---
 tubesync/sync/models.py | 43 ++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 54fcdaa6..76dea0b1 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1144,28 +1144,35 @@ class Media(models.Model):
         return self.metadata is not None
 
 
-    def reduce_data(self, data):
-        from common.logger import log
-        from common.utils import json_serial
-        # log the results of filtering / compacting on metadata size
-        filtered_data = filter_response(data)
-        compact_metadata = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
-        old_mdl = len(self.metadata)
-        new_mdl = len(compact_metadata)
-        if old_mdl > new_mdl:
-            delta = old_mdl - new_mdl
-            log.info(f'{self.key}: metadata reduced by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')
+    @property
+    def reduce_data(self):
+        try:
+            from common.logger import log
+            from common.utils import json_serial
+
+            old_mdl = len(self.metadata or "")
+            data = json.loads(self.metadata or "")
+            compact_data = json.dumps(data, separators=(',', ':'), default=json_serial)
+            
+            filtered_data = filter_response(data)
+            filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
+        except Exception as e:
+            log.exception('reduce_data: %s', e)
+        else:
+            # log the results of filtering / compacting on metadata size
+            new_mdl = len(compact_data)
+            if old_mdl > new_mdl:
+                delta = old_mdl - new_mdl
+                log.info(f'{self.key}: metadata compacted by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')
+            new_mdl = len(filtered_json)
+            if old_mdl > new_mdl:
+                delta = old_mdl - new_mdl
+                log.info(f'{self.key}: metadata reduced by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')
 
 
     @property
     def loaded_metadata(self):
-        from common.logger import log
-        try:
-            self.reduce_data(json.loads(self.metadata))
-        except Exception as e:
-            log.exception('reduce_data: %s', e)
-            pass
-
+        self.reduce_data
         try:
             data = json.loads(self.metadata)
             if not isinstance(data, dict):

From 100382f66fea8b8dd27532932f23f4160d354401 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 09:28:58 -0500
Subject: [PATCH 054/143] Rename compact_data to compact_json

This was misleading because the data dict becomes a JSON string.
---
 tubesync/sync/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 76dea0b1..67453f03 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1152,7 +1152,7 @@ class Media(models.Model):
 
             old_mdl = len(self.metadata or "")
             data = json.loads(self.metadata or "")
-            compact_data = json.dumps(data, separators=(',', ':'), default=json_serial)
+            compact_json = json.dumps(data, separators=(',', ':'), default=json_serial)
             
             filtered_data = filter_response(data)
             filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
@@ -1160,7 +1160,7 @@ class Media(models.Model):
             log.exception('reduce_data: %s', e)
         else:
             # log the results of filtering / compacting on metadata size
-            new_mdl = len(compact_data)
+            new_mdl = len(compact_json)
             if old_mdl > new_mdl:
                 delta = old_mdl - new_mdl
                 log.info(f'{self.key}: metadata compacted by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')

From 682a53da34d18d777e58e6080df4390f44519686 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 10:17:37 -0500
Subject: [PATCH 055/143] Add a filter_response test

First, only check that changes did happen.
---
 tubesync/sync/tests.py | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 8f0de6ef..935ad569 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -18,6 +18,7 @@ from background_task.models import Task
 from .models import Source, Media
 from .tasks import cleanup_old_media
 from .filtering import filter_media
+from .utils import filter_response
 
 
 class FrontEndTestCase(TestCase):
@@ -1709,6 +1710,43 @@ class FormatMatchingTestCase(TestCase):
                     f'expected {expected_match_result}')
 
 
+class ResponseFilteringTestCase(TestCase):
+
+    def setUp(self):
+        # Disable general logging for test case
+        logging.disable(logging.CRITICAL)
+        # Add a test source
+        self.source = Source.objects.create(
+            source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
+            key='testkey',
+            name='testname',
+            directory='testdirectory',
+            index_schedule=3600,
+            delete_old_media=False,
+            days_to_keep=14,
+            source_resolution=Source.SOURCE_RESOLUTION_1080P,
+            source_vcodec=Source.SOURCE_VCODEC_VP9,
+            source_acodec=Source.SOURCE_ACODEC_OPUS,
+            prefer_60fps=False,
+            prefer_hdr=False,
+            fallback=Source.FALLBACK_FAIL
+        )
+        # Add some media
+        self.media = Media.objects.create(
+            key='mediakey',
+            source=self.source,
+            metadata='{}'
+        )
+
+    def test_metadata_20230629(self):
+        self.media.metadata = all_test_metadata['20230629']
+        self.media.save()
+
+        unfiltered = self.media.loaded_metadata
+        filtered = filter_response(self.media.loaded_metadata)
+        self.assertNotEqual(len(str(unfiltered)), len(str(filtered)))
+
+
 class TasksTestCase(TestCase):
 
     def setUp(self):

From 4c9fa40bb0e47871caffaf9a3212932727ffc1cb Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 11:47:10 -0500
Subject: [PATCH 056/143] More filter_response asserts

---
 tubesync/sync/tests.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 935ad569..bc199282 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -1744,7 +1744,41 @@ class ResponseFilteringTestCase(TestCase):
 
         unfiltered = self.media.loaded_metadata
         filtered = filter_response(self.media.loaded_metadata)
-        self.assertNotEqual(len(str(unfiltered)), len(str(filtered)))
+        self.assertIn('formats', unfiltered.keys())
+        self.assertIn('formats', filtered.keys())
+        # filtered 'http_headers'
+        self.assertIn('http_headers', unfiltered['formats'][0].keys())
+        self.assertNotIn('http_headers', filtered['formats'][0].keys())
+        # did not lose any formats
+        self.assertEqual(48, len(unfiltered['formats']))
+        self.assertEqual(48, len(filtered['formats']))
+        self.assertEqual(len(unfiltered['formats']), len(filtered['formats']))
+        # did reduce the size of the metadata
+        self.assertTrue(len(str(filtered)) < len(str(unfiltered)))
+
+        url_keys = []
+        for format in unfiltered['formats']:
+            for key in format.keys():
+                if 'url' in key:
+                    url_keys.append((format['format_id'], key, format[key],))
+        unfiltered_url_keys = url_keys
+        self.assertEqual(63, len(unfiltered_url_keys), msg=str(unfiltered_url_keys))
+
+        url_keys = []
+        for format in filtered['formats']:
+            for key in format.keys():
+                if 'url' in key:
+                    url_keys.append((format['format_id'], key, format[key],))
+        filtered_url_keys = url_keys
+        self.assertEqual(3, len(filtered_url_keys), msg=str(filtered_url_keys))
+
+        url_keys = []
+        for lang_code, captions in filtered['automatic_captions'].items():
+            for caption in captions:
+                for key in caption.keys():
+                    if 'url' in key:
+                        url_keys.append((lang_code, caption['ext'], caption[key],))
+        self.assertEqual(0, len(url_keys), msg=str(url_keys))
 
 
 class TasksTestCase(TestCase):

From 3e3f80d287c637c34f5c5094aa313531dfbe7b77 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 12:04:01 -0500
Subject: [PATCH 057/143] More filter_response asserts

---
 tubesync/sync/tests.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index bc199282..2704058f 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -1746,6 +1746,9 @@ class ResponseFilteringTestCase(TestCase):
         filtered = filter_response(self.media.loaded_metadata)
         self.assertIn('formats', unfiltered.keys())
         self.assertIn('formats', filtered.keys())
+        # filtered 'downloader_options'
+        self.assertIn('downloader_options', unfiltered['formats'][10].keys())
+        self.assertNotIn('downloader_options', filtered['formats'][10].keys())
         # filtered 'http_headers'
         self.assertIn('http_headers', unfiltered['formats'][0].keys())
         self.assertNotIn('http_headers', filtered['formats'][0].keys())
@@ -1753,6 +1756,10 @@ class ResponseFilteringTestCase(TestCase):
         self.assertEqual(48, len(unfiltered['formats']))
         self.assertEqual(48, len(filtered['formats']))
         self.assertEqual(len(unfiltered['formats']), len(filtered['formats']))
+        # did not remove everything with url
+        self.assertIn('original_url', unfiltered.keys())
+        self.assertIn('original_url', filtered.keys())
+        self.assertEqual(unfiltered['original_url'], filtered['original_url'])
         # did reduce the size of the metadata
         self.assertTrue(len(str(filtered)) < len(str(unfiltered)))
 

From 29c39aab1f7096a7267c351cc3ebf0d786c98723 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 13:20:22 -0500
Subject: [PATCH 058/143] Add SHRINK_NEW_MEDIA_METADATA setting

---
 tubesync/sync/tasks.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 30f8c827..644918b7 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -8,6 +8,7 @@ import os
 import json
 import math
 import uuid
+from copy import deepcopy
 from io import BytesIO
 from hashlib import sha1
 from datetime import timedelta, datetime
@@ -26,7 +27,7 @@ from common.errors import NoMediaException, DownloadFailedException
 from common.utils import json_serial
 from .models import Source, Media, MediaServer
 from .utils import (get_remote_image, resize_image_to_height, delete_file,
-                    write_text_file)
+                    write_text_file, filter_response)
 from .filtering import filter_media
 
 
@@ -304,7 +305,11 @@ def download_media_metadata(media_id):
         return
     source = media.source
     metadata = media.index_metadata()
-    media.metadata = json.dumps(metadata, separators=(',', ':'), default=json_serial)
+    if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False):
+        response = filter_response(deepcopy(metadata))
+    else:
+        response = metadata
+    media.metadata = json.dumps(response, separators=(',', ':'), default=json_serial)
     upload_date = media.upload_date
     # Media must have a valid upload date
     if upload_date:

From 0f986949e5ad18195de2265eae83f5360f6c5277 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 13:36:43 -0500
Subject: [PATCH 059/143] Have filter_response return a copy, if requested

---
 tubesync/sync/utils.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index b424528b..1d67af38 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -1,6 +1,7 @@
 import os
 import re
 import math
+from copy import deepcopy
 from operator import itemgetter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
@@ -189,13 +190,18 @@ def _drop_url_keys(arg_dict, key, filter_func):
                     del val_dict[url_key]
 
 
-def filter_response(response_dict):
+def filter_response(arg_dict, copy_arg=False):
     '''
         Clean up the response so as to not store useless metadata in the database.
     '''
+    response_dict = arg_dict
     # raise an exception for an unexpected argument type
     if not isinstance(response_dict, dict):
         raise TypeError(f'response_dict must be a dict, got "{type(response_dict)}"')
+
+    if copy_arg:
+        response_dict = deepcopy(arg_dict)
+
     # optimize the empty case
     if not response_dict:
         return response_dict

From 274f19fa15547c1a9d76c967e4134ffafa822aa1 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 13:41:23 -0500
Subject: [PATCH 060/143] Use the new copy argument to filter_response

---
 tubesync/sync/tasks.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 644918b7..ab92e2c8 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -8,7 +8,6 @@ import os
 import json
 import math
 import uuid
-from copy import deepcopy
 from io import BytesIO
 from hashlib import sha1
 from datetime import timedelta, datetime
@@ -305,10 +304,9 @@ def download_media_metadata(media_id):
         return
     source = media.source
     metadata = media.index_metadata()
+    response = metadata
     if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False):
-        response = filter_response(deepcopy(metadata))
-    else:
-        response = metadata
+        response = filter_response(metadata, True)
     media.metadata = json.dumps(response, separators=(',', ':'), default=json_serial)
     upload_date = media.upload_date
     # Media must have a valid upload date

From 1ff8dfda9897dd8c409feba2649b5ce15f5f7e32 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 13:53:12 -0500
Subject: [PATCH 061/143] Use the new copy argument to filter_response

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 67453f03..10fbbdbd 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1154,7 +1154,7 @@ class Media(models.Model):
             data = json.loads(self.metadata or "")
             compact_json = json.dumps(data, separators=(',', ':'), default=json_serial)
             
-            filtered_data = filter_response(data)
+            filtered_data = filter_response(data, True)
             filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
         except Exception as e:
             log.exception('reduce_data: %s', e)

From 6292a9a59dc5d05db79241b9bd2d58f51be3cc6a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:22:37 -0500
Subject: [PATCH 062/143] Add SHRINK_OLD_MEDIA_METADATA setting

---
 tubesync/sync/models.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 10fbbdbd..bb850af3 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1168,6 +1168,8 @@ class Media(models.Model):
             if old_mdl > new_mdl:
                 delta = old_mdl - new_mdl
                 log.info(f'{self.key}: metadata reduced by {delta:,} characters ({old_mdl:,} -> {new_mdl:,})')
+                if getattr(settings, 'SHRINK_OLD_MEDIA_METADATA', False):
+                    self.metadata = filtered_json
 
 
     @property

From 81edd08c7d8ce8d0844b82751b730d2dc91ff4ac Mon Sep 17 00:00:00 2001
From: Makhuta <matykubla@gmail.com>
Date: Sat, 11 Jan 2025 14:38:31 +0100
Subject: [PATCH 063/143] Update

- added video order to Media Format
---
 tubesync/sync/models.py                            | 9 +++++++--
 tubesync/sync/templates/sync/_mediaformatvars.html | 5 +++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 2037492d..8e37bdbe 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -589,6 +589,7 @@ class Source(models.Model):
             'key': 'SoMeUnIqUiD',
             'format': '-'.join(fmt),
             'playlist_title': 'Some Playlist Title',
+            'video_order': '1',
             'ext': self.extension,
             'resolution': self.source_resolution if self.source_resolution else '',
             'height': '720' if self.source_resolution else '',
@@ -1128,6 +1129,7 @@ class Media(models.Model):
             'key': self.key,
             'format': '-'.join(display_format['format']),
             'playlist_title': self.playlist_title,
+            'video_order': self.get_episode_str(),
             'ext': self.source.extension,
             'resolution': display_format['resolution'],
             'height': display_format['height'],
@@ -1373,8 +1375,7 @@ class Media(models.Model):
         nfo.append(season)
         # episode = number of video in the year
         episode = nfo.makeelement('episode', {})
-        episode_number = self.calculate_episode_number()
-        episode.text = str(episode_number) if episode_number else ''
+        episode.text = self.get_episode_str()
         episode.tail = '\n  '
         nfo.append(episode)
         # ratings = media metadata youtube rating
@@ -1524,6 +1525,10 @@ class Media(models.Model):
                 return position_counter
             position_counter += 1
 
+    def get_episode_str(self):
+        episode_number = self.calculate_episode_number()
+        return f'{episode_number:02}' if episode_number else ''
+
 
 class MediaServer(models.Model):
     '''
diff --git a/tubesync/sync/templates/sync/_mediaformatvars.html b/tubesync/sync/templates/sync/_mediaformatvars.html
index 438b200a..06068f90 100644
--- a/tubesync/sync/templates/sync/_mediaformatvars.html
+++ b/tubesync/sync/templates/sync/_mediaformatvars.html
@@ -73,6 +73,11 @@
       <td>Playlist title of media, if it's in a playlist</td>
       <td>Some Playlist</td>
     </tr>
+    <tr>
+      <td>{video_order}</td>
+      <td>Episode order in playlist, if in playlist <sub><sup>(can cause issues if playlist is changed after adding)</sup></sub></td>
+      <td>01</td>
+    </tr>
     <tr>
       <td>{ext}</td>
       <td>File extension</td>

From 8dda325dbd841535708ca8f8d58602d26080b019 Mon Sep 17 00:00:00 2001
From: Makhuta <matykubla@gmail.com>
Date: Sat, 11 Jan 2025 15:53:36 +0100
Subject: [PATCH 064/143] Update models.py

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 8e37bdbe..a5b7adbd 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -589,7 +589,7 @@ class Source(models.Model):
             'key': 'SoMeUnIqUiD',
             'format': '-'.join(fmt),
             'playlist_title': 'Some Playlist Title',
-            'video_order': '1',
+            'video_order': '01',
             'ext': self.extension,
             'resolution': self.source_resolution if self.source_resolution else '',
             'height': '720' if self.source_resolution else '',

From 4364ebbff3cd2f8147206ce05c63745cda88406c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 11 Jan 2025 11:53:10 -0500
Subject: [PATCH 065/143] Multi-stage docker build for ffmpeg & s6-overlay

* Create a s6-overlay-extracted stage to copy from

This was largely inspired by: @socheatsok78

Our downloaded files are checked where that version doesn't do any verification of the downloads.

* Update ffmpeg to the first build with checksums.sha256

* Create a ffmpeg-extracted stage to copy from

* Don't preserve ownership from the builder

I was sick of the extra work with ffmpeg builds.

So, I managed to get sums generated for those builds and now we don't need to manually fill out SHA256 hashes anymore.

Now to bump ffmpeg, we can just change the date.
---
 Dockerfile | 286 +++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 202 insertions(+), 84 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 880dd677..a69609c5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,17 +1,202 @@
-FROM debian:bookworm-slim
-
-ARG TARGETARCH
-ARG TARGETPLATFORM
+ARG FFMPEG_DATE="2025-01-10-19-43"
+ARG FFMPEG_VERSION="N-118280-g5cd49e1bfd"
 
 ARG S6_VERSION="3.2.0.2"
+
 ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478"
 ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785"
 ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
 
-ARG FFMPEG_DATE="autobuild-2024-12-24-14-15"
-ARG FFMPEG_VERSION="N-118163-g954d55c2a4"
-ARG SHA256_FFMPEG_AMD64="798a7e5a0724139e6bb70df8921522b23be27028f9f551dfa83c305ec4ffaf3a"
-ARG SHA256_FFMPEG_ARM64="c3e6cc0fec42cc7e3804014fbb02c1384a1a31ef13f6f9a36121f2e1216240c0"
+ARG ALPINE_VERSION="latest"
+ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION%%-*}"
+ARG FFMPEG_SUFFIX_FILE=".tar.xz"
+
+FROM alpine:${ALPINE_VERSION} AS ffmpeg-download
+ARG FFMPEG_DATE
+ARG FFMPEG_VERSION
+ARG FFMPEG_PREFIX_FILE
+ARG FFMPEG_SUFFIX_FILE
+ARG SHA256_FFMPEG_AMD64
+ARG SHA256_FFMPEG_ARM64
+ARG CHECKSUM_ALGORITHM="sha256"
+ARG FFMPEG_CHECKSUM_AMD64="${SHA256_FFMPEG_AMD64}"
+ARG FFMPEG_CHECKSUM_ARM64="${SHA256_FFMPEG_ARM64}"
+
+ARG FFMPEG_FILE_SUMS="checksums.${CHECKSUM_ALGORITHM}"
+ARG FFMPEG_URL="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-${FFMPEG_DATE}"
+
+ARG DESTDIR="/downloaded"
+ARG TARGETARCH
+ADD "${FFMPEG_URL}/${FFMPEG_FILE_SUMS}" "${DESTDIR}/"
+RUN set -eu ; \
+    apk --no-cache --no-progress add cmd:aria2c cmd:awk ; \
+\
+    aria2c_options() { \
+        algorithm="${CHECKSUM_ALGORITHM%[0-9]??}" ; \
+        bytes="${CHECKSUM_ALGORITHM#${algorithm}}" ; \
+        hash="$( awk -v fn="${1##*/}" '$0 ~ fn"$" { print $1; exit; }' "${DESTDIR}/${FFMPEG_FILE_SUMS}" )" ; \
+\
+        printf -- '\t%s\n' \
+          'allow-overwrite=true' \
+          'always-resume=false' \
+          'check-integrity=true' \
+          "checksum=${algorithm}-${bytes}=${hash}" \
+          'max-connection-per-server=2' \
+; \
+        printf -- '\n' ; \
+    } ; \
+\
+    decide_arch() { \
+        case "${TARGETARCH}" in \
+            (amd64) printf -- 'linux64' ;; \
+            (arm64) printf -- 'linuxarm64' ;; \
+        esac ; \
+    } ; \
+\
+    FFMPEG_ARCH="$(decide_arch)" ; \
+    for url in $(awk ' \
+      $2 ~ /^[*]?'"${FFMPEG_PREFIX_FILE}"'/ && /-'"${FFMPEG_ARCH}"'-/ { $1=""; print; } \
+      ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \
+    do \
+        url="${FFMPEG_URL}/${url# }" ; \
+        printf -- '%s\n' "${url}" ; \
+        aria2c_options "${url}" ; \
+        printf -- '\n' ; \
+    done > /tmp/downloads ; \
+    unset -v url ; \
+\
+    aria2c --no-conf=true \
+      --dir /downloaded \
+      --lowest-speed-limit='16K' \
+      --show-console-readout=false \
+      --summary-interval=0 \
+      --input-file /tmp/downloads ; \
+\
+    apk --no-cache --no-progress add cmd:awk "cmd:${CHECKSUM_ALGORITHM}sum" ; \
+\    
+    decide_expected() { \
+        case "${TARGETARCH}" in \
+            (amd64) printf -- '%s' "${FFMPEG_CHECKSUM_AMD64}" ;; \
+            (arm64) printf -- '%s' "${FFMPEG_CHECKSUM_ARM64}" ;; \
+        esac ; \
+    } ; \
+\
+    FFMPEG_HASH="$(decide_expected)" ; \
+\
+    cd "${DESTDIR}" ; \
+    if [ -n "${FFMPEG_HASH}" ] ; \
+    then \
+        printf -- '%s *%s\n' "${FFMPEG_HASH}" "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" >> /tmp/SUMS ; \
+        "${CHECKSUM_ALGORITHM}sum" --check --strict /tmp/SUMS || exit ; \
+    fi ; \
+    "${CHECKSUM_ALGORITHM}sum" --check --strict --ignore-missing "${DESTDIR}/${FFMPEG_FILE_SUMS}" ; \
+\
+    mkdir -v -p "/verified/${TARGETARCH}" ; \
+    ln -v "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" "/verified/${TARGETARCH}/" ; \
+    rm -rf "${DESTDIR}" ;
+
+FROM alpine:${ALPINE_VERSION} AS ffmpeg-extracted
+COPY --link --from=ffmpeg-download /verified /verified
+
+ARG FFMPEG_PREFIX_FILE
+ARG FFMPEG_SUFFIX_FILE
+ARG TARGETARCH
+RUN set -eu ; \
+    apk --no-cache --no-progress add cmd:tar cmd:xz ; \
+\
+    mkdir -v /extracted ; \
+    cd /extracted ; \
+    set -x ; \
+    tar -xp \
+      --strip-components=2 \
+      --no-anchored \
+      --no-same-owner \
+      -f "/verified/${TARGETARCH}"/"${FFMPEG_PREFIX_FILE}"*"${FFMPEG_SUFFIX_FILE}" \
+      'ffmpeg' 'ffprobe' ; \
+\
+    ls -AlR /extracted ;
+
+FROM scratch AS s6-overlay-download
+ARG S6_VERSION
+ARG SHA256_S6_AMD64
+ARG SHA256_S6_ARM64
+ARG SHA256_S6_NOARCH
+
+ARG DESTDIR="/downloaded"
+ARG CHECKSUM_ALGORITHM="sha256"
+
+ARG S6_CHECKSUM_AMD64="${CHECKSUM_ALGORITHM}:${SHA256_S6_AMD64}"
+ARG S6_CHECKSUM_ARM64="${CHECKSUM_ALGORITHM}:${SHA256_S6_ARM64}"
+ARG S6_CHECKSUM_NOARCH="${CHECKSUM_ALGORITHM}:${SHA256_S6_NOARCH}"
+
+ARG S6_OVERLAY_URL="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}"
+ARG S6_PREFIX_FILE="s6-overlay-"
+ARG S6_SUFFIX_FILE=".tar.xz"
+
+ARG S6_FILE_AMD64="${S6_PREFIX_FILE}x86_64${S6_SUFFIX_FILE}"
+ARG S6_FILE_ARM64="${S6_PREFIX_FILE}aarch64${S6_SUFFIX_FILE}"
+ARG S6_FILE_NOARCH="${S6_PREFIX_FILE}noarch${S6_SUFFIX_FILE}"
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+
+ADD --checksum="${S6_CHECKSUM_AMD64}" "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
+ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
+ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+
+FROM alpine:${ALPINE_VERSION} AS s6-overlay-extracted
+COPY --link --from=s6-overlay-download /downloaded /downloaded
+
+ARG TARGETARCH
+
+RUN set -eu ; \
+\
+    decide_arch() { \
+      local arg1 ; \
+      arg1="${1:-$(uname -m)}" ; \
+\
+      case "${arg1}" in \
+        (amd64) printf -- 'x86_64' ;; \
+        (arm64) printf -- 'aarch64' ;; \
+        (armv7l) printf -- 'arm' ;; \
+        (*) printf -- '%s' "${arg1}" ;; \
+      esac ; \
+      unset -v arg1 ; \
+    } ; \
+\
+    mkdir -v /verified ; \
+    cd /downloaded ; \
+    for f in *.sha256 ; \
+    do \
+      sha256sum -c < "${f}" || exit ; \
+      ln -v "${f%.sha256}" /verified/ || exit ; \
+    done ; \
+    unset -v f ; \
+\
+    S6_ARCH="$(decide_arch "${TARGETARCH}")" ; \
+    set -x ; \
+    mkdir -v /s6-overlay-rootfs ; \
+    cd /s6-overlay-rootfs ; \
+    for f in /verified/*.tar* ; \
+    do \
+      case "${f}" in \
+        (*-noarch.tar*|*-"${S6_ARCH}".tar*) \
+          tar -xpf "${f}" || exit ;; \
+      esac ; \
+    done ; \
+    set +x ; \
+    unset -v f ;
+
+FROM debian:bookworm-slim AS tubesync
+
+ARG TARGETARCH
+ARG TARGETPLATFORM
+
+ARG S6_VERSION
+
+ARG FFMPEG_DATE
+ARG FFMPEG_VERSION
 
 ENV S6_VERSION="${S6_VERSION}" \
   FFMPEG_DATE="${FFMPEG_DATE}" \
@@ -26,89 +211,20 @@ ENV DEBIAN_FRONTEND="noninteractive" \
   S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
 
 # Install third party software
+COPY --link --from=s6-overlay-extracted /s6-overlay-rootfs /
+COPY --link --from=ffmpeg-extracted /extracted /usr/local/bin/
+
 # Reminder: the SHELL handles all variables
-RUN decide_arch() { \
-      case "${TARGETARCH:=amd64}" in \
-        (arm64) printf -- 'aarch64' ;; \
-        (*) printf -- '%s' "${TARGETARCH}" ;; \
-      esac ; \
-    } && \
-    decide_expected() { \
-      case "${1}" in \
-        (ffmpeg) case "${2}" in \
-            (amd64) printf -- '%s' "${SHA256_FFMPEG_AMD64}" ;; \
-            (arm64) printf -- '%s' "${SHA256_FFMPEG_ARM64}" ;; \
-          esac ;; \
-        (s6) case "${2}" in \
-            (amd64) printf -- '%s' "${SHA256_S6_AMD64}" ;; \
-            (arm64) printf -- '%s' "${SHA256_S6_ARM64}" ;; \
-            (noarch) printf -- '%s' "${SHA256_S6_NOARCH}" ;; \
-          esac ;; \
-      esac ; \
-    } && \
-    decide_url() { \
-      case "${1}" in \
-        (ffmpeg) printf -- \
-          'https://github.com/yt-dlp/FFmpeg-Builds/releases/download/%s/ffmpeg-%s-linux%s-gpl%s.tar.xz' \
-          "${FFMPEG_DATE}" \
-          "${FFMPEG_VERSION}" \
-          "$(case "${2}" in \
-            (amd64) printf -- '64' ;; \
-            (*) printf -- '%s' "${2}" ;; \
-          esac)" \
-          "$(case "${FFMPEG_VERSION%%-*}" in \
-            (n*) printf -- '-%s\n' "${FFMPEG_VERSION#n}" | cut -d '-' -f 1,2 ;; \
-            (*) printf -- '' ;; \
-          esac)" ;; \
-        (s6) printf -- \
-          'https://github.com/just-containers/s6-overlay/releases/download/v%s/s6-overlay-%s.tar.xz' \
-          "${S6_VERSION}" \
-          "$(case "${2}" in \
-            (amd64) printf -- 'x86_64' ;; \
-            (arm64) printf -- 'aarch64' ;; \
-            (*) printf -- '%s' "${2}" ;; \
-          esac)" ;; \
-      esac ; \
-    } && \
-    verify_download() { \
-      while [ $# -ge 2 ] ; do \
-        sha256sum "${2}" ; \
-        printf -- '%s  %s\n' "${1}" "${2}" | sha256sum -c || return ; \
-        shift ; shift ; \
-      done ; \
-    } && \
-    download_expected_file() { \
-      local arg1 expected file url ; \
-      arg1="$(printf -- '%s\n' "${1}" | awk '{print toupper($0);}')" ; \
-      expected="$(decide_expected "${1}" "${2}")" ; \
-      file="${3}" ; \
-      url="$(decide_url "${1}" "${2}")" ; \
-      printf -- '%s\n' \
-        "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \
-      rm -rf "${file}" && \
-      curl --disable --output "${file}" --clobber --location --no-progress-meter --url "${url}" && \
-      verify_download "${expected}" "${file}" ; \
-    } && \
-  export ARCH="$(decide_arch)" && \
-  set -x && \
+RUN set -x && \
   apt-get update && \
   apt-get -y --no-install-recommends install locales && \
   printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \
   locale-gen en_US.UTF-8 && \
   # Install required distro packages
   apt-get -y --no-install-recommends install curl ca-certificates file binutils xz-utils && \
-  # Install s6
-  _file="/tmp/s6-overlay-noarch.tar.xz" && \
-  download_expected_file s6 noarch "${_file}" && \
-  tar -C / -xpf "${_file}" && rm -f "${_file}" && \
-  _file="/tmp/s6-overlay-${ARCH}.tar.xz" && \
-  download_expected_file s6 "${TARGETARCH}" "${_file}" && \
-  tar -C / -xpf "${_file}" && rm -f "${_file}" && \
+  # Installed s6 (using COPY earlier)
   file -L /command/s6-overlay-suexec && \
-  # Install ffmpeg
-  _file="/tmp/ffmpeg-${ARCH}.tar.xz" && \
-  download_expected_file ffmpeg "${TARGETARCH}" "${_file}" && \
-  tar -xvvpf "${_file}" --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" "ffprobe" && rm -f "${_file}" && \
+  # Installed ffmpeg (using COPY earlier)
   /usr/local/bin/ffmpeg -version && \
   file /usr/local/bin/ff* && \
   # Clean up
@@ -154,7 +270,9 @@ ENV PIP_NO_COMPILE=1 \
 WORKDIR /app
 
 # Set up the app
-RUN set -x && \
+#BuildKit#RUN --mount=type=bind,source=Pipfile,target=/app/Pipfile \
+RUN \
+  set -x && \
   apt-get update && \
   # Install required build packages
   apt-get -y --no-install-recommends install \

From f464acaa6331913abd5bb341344568f6d9eb73fc Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 11 Jan 2025 15:38:45 -0500
Subject: [PATCH 066/143] Simplify directory_path for Media

---
 tubesync/sync/models.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 2037492d..ad17258c 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1263,8 +1263,7 @@ class Media(models.Model):
 
     @property
     def directory_path(self):
-        dirname = self.source.directory_path / self.filename
-        return dirname.parent
+        return self.filepath.parent
 
     @property
     def filepath(self):

From 3ea7e6c8ee0ab7631507938734c255ec9116c2bb Mon Sep 17 00:00:00 2001
From: Makhuta <matykubla@gmail.com>
Date: Sat, 11 Jan 2025 22:07:36 +0100
Subject: [PATCH 067/143] Change

- changed the episode_str to be togglable and use the old format by default
---
 tubesync/sync/models.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index a5b7adbd..d22cdb57 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1129,7 +1129,7 @@ class Media(models.Model):
             'key': self.key,
             'format': '-'.join(display_format['format']),
             'playlist_title': self.playlist_title,
-            'video_order': self.get_episode_str(),
+            'video_order': self.get_episode_str(True),
             'ext': self.source.extension,
             'resolution': display_format['resolution'],
             'height': display_format['height'],
@@ -1525,9 +1525,12 @@ class Media(models.Model):
                 return position_counter
             position_counter += 1
 
-    def get_episode_str(self):
+    def get_episode_str(self, use_padding=False):
         episode_number = self.calculate_episode_number()
-        return f'{episode_number:02}' if episode_number else ''
+        if use_padding:
+            return f'{episode_number:02}' if episode_number else ''
+        
+        return str(episode_number) if episode_number else ''
 
 
 class MediaServer(models.Model):

From df4b824672bcc00442064ced9de6b86e05a505ea Mon Sep 17 00:00:00 2001
From: Makhuta <matykubla@gmail.com>
Date: Sat, 11 Jan 2025 22:17:10 +0100
Subject: [PATCH 068/143] Change

- simplified the returns
---
 tubesync/sync/models.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index d22cdb57..66bb0481 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1527,10 +1527,13 @@ class Media(models.Model):
 
     def get_episode_str(self, use_padding=False):
         episode_number = self.calculate_episode_number()
+        if not episode_number:
+            return ''
+
         if use_padding:
-            return f'{episode_number:02}' if episode_number else ''
+            return f'{episode_number:02}'
         
-        return str(episode_number) if episode_number else ''
+        return str(episode_number)
 
 
 class MediaServer(models.Model):

From ef4181c2c42239512c811bdb6fb456bdcb0289cd Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 12 Jan 2025 00:37:30 -0500
Subject: [PATCH 069/143] Dockerfile syntax and checks

- Specify the syntax be the latest stable version and that failed checks should stop the build.

```
By default, builds with failing build checks exit with a zero status code despite warnings. To make the build fail on warnings, set #check=error=true.
```

- Use the form of health checking that doesn't involve an extra shell on every check.
---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 880dd677..c63e24d2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,6 @@
+# syntax=docker/dockerfile:1
+# check=error=true
+
 FROM debian:bookworm-slim
 
 ARG TARGETARCH
@@ -228,7 +231,7 @@ RUN set -x && \
 COPY config/root /
 
 # Create a healthcheck
-HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck
+HEALTHCHECK --interval=1m --timeout=10s --start-period=3m CMD ["/app/healthcheck.py", "http://127.0.0.1:8080/healthcheck"]
 
 # ENVS and ports
 ENV PYTHONPATH="/app" PYTHONPYCACHEPREFIX="/config/cache/pycache"

From 5e5d011b640be82d7c5d7d749f1801be787c46bf Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 12 Jan 2025 00:49:19 -0500
Subject: [PATCH 070/143] Add parser directives

This hopefully helps anyone building on an older docker, such as Debian / Ubuntu packaged versions.
---
 Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index a69609c5..f7a26bb3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,6 @@
+# syntax=docker/dockerfile:1
+# check=error=true
+
 ARG FFMPEG_DATE="2025-01-10-19-43"
 ARG FFMPEG_VERSION="N-118280-g5cd49e1bfd"
 

From 2860147212fb21087d699761d959727aae1c707a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 12 Jan 2025 04:46:32 -0500
Subject: [PATCH 071/143] Build on older docker also

* Do without --link for COPY or ADD

* Do without --checksum for ADD

* Trim the FFMPEG_VERSION variable with cut instead

I've built successfully on old Debian systems using these changes. Everything else I use has a newer docker on it.
---
 Dockerfile | 74 ++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 50 insertions(+), 24 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f7a26bb3..d0107385 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,9 +11,12 @@ ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f6
 ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
 
 ARG ALPINE_VERSION="latest"
-ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION%%-*}"
+ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION}"
 ARG FFMPEG_SUFFIX_FILE=".tar.xz"
 
+ARG FFMPEG_CHECKSUM_ALGORITHM="sha256"
+ARG S6_CHECKSUM_ALGORITHM="sha256"
+
 FROM alpine:${ALPINE_VERSION} AS ffmpeg-download
 ARG FFMPEG_DATE
 ARG FFMPEG_VERSION
@@ -21,7 +24,8 @@ ARG FFMPEG_PREFIX_FILE
 ARG FFMPEG_SUFFIX_FILE
 ARG SHA256_FFMPEG_AMD64
 ARG SHA256_FFMPEG_ARM64
-ARG CHECKSUM_ALGORITHM="sha256"
+ARG FFMPEG_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${FFMPEG_CHECKSUM_ALGORITHM}"
 ARG FFMPEG_CHECKSUM_AMD64="${SHA256_FFMPEG_AMD64}"
 ARG FFMPEG_CHECKSUM_ARM64="${SHA256_FFMPEG_ARM64}"
 
@@ -57,6 +61,7 @@ RUN set -eu ; \
     } ; \
 \
     FFMPEG_ARCH="$(decide_arch)" ; \
+    FFMPEG_PREFIX_FILE="$( printf -- '%s' "${FFMPEG_PREFIX_FILE}" | cut -d '-' -f 1,2 )" ; \
     for url in $(awk ' \
       $2 ~ /^[*]?'"${FFMPEG_PREFIX_FILE}"'/ && /-'"${FFMPEG_ARCH}"'-/ { $1=""; print; } \
       ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \
@@ -75,7 +80,7 @@ RUN set -eu ; \
       --summary-interval=0 \
       --input-file /tmp/downloads ; \
 \
-    apk --no-cache --no-progress add cmd:awk "cmd:${CHECKSUM_ALGORITHM}sum" ; \
+    apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \
 \    
     decide_expected() { \
         case "${TARGETARCH}" in \
@@ -90,43 +95,44 @@ RUN set -eu ; \
     if [ -n "${FFMPEG_HASH}" ] ; \
     then \
         printf -- '%s *%s\n' "${FFMPEG_HASH}" "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" >> /tmp/SUMS ; \
-        "${CHECKSUM_ALGORITHM}sum" --check --strict /tmp/SUMS || exit ; \
+        "${CHECKSUM_ALGORITHM}sum" --check --warn --strict /tmp/SUMS || exit ; \
     fi ; \
-    "${CHECKSUM_ALGORITHM}sum" --check --strict --ignore-missing "${DESTDIR}/${FFMPEG_FILE_SUMS}" ; \
+    "${CHECKSUM_ALGORITHM}sum" --check --warn --strict --ignore-missing "${DESTDIR}/${FFMPEG_FILE_SUMS}" ; \
 \
     mkdir -v -p "/verified/${TARGETARCH}" ; \
     ln -v "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" "/verified/${TARGETARCH}/" ; \
     rm -rf "${DESTDIR}" ;
 
 FROM alpine:${ALPINE_VERSION} AS ffmpeg-extracted
-COPY --link --from=ffmpeg-download /verified /verified
+COPY --from=ffmpeg-download /verified /verified
 
 ARG FFMPEG_PREFIX_FILE
 ARG FFMPEG_SUFFIX_FILE
 ARG TARGETARCH
-RUN set -eu ; \
-    apk --no-cache --no-progress add cmd:tar cmd:xz ; \
-\
+RUN set -eux ; \
     mkdir -v /extracted ; \
     cd /extracted ; \
-    set -x ; \
-    tar -xp \
+    ln -s "/verified/${TARGETARCH}"/"${FFMPEG_PREFIX_FILE}"*"${FFMPEG_SUFFIX_FILE}" "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" ; \
+    tar -tf "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" | grep '/bin/\(ffmpeg\|ffprobe\)' > /tmp/files ; \
+    tar -xop \
       --strip-components=2 \
-      --no-anchored \
-      --no-same-owner \
-      -f "/verified/${TARGETARCH}"/"${FFMPEG_PREFIX_FILE}"*"${FFMPEG_SUFFIX_FILE}" \
-      'ffmpeg' 'ffprobe' ; \
+      -f "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" \
+      -T /tmp/files ; \
 \
     ls -AlR /extracted ;
 
-FROM scratch AS s6-overlay-download
+FROM scratch AS ffmpeg
+COPY --from=ffmpeg-extracted /extracted /usr/local/bin/
+
+FROM alpine:${ALPINE_VERSION} AS s6-overlay-download
 ARG S6_VERSION
 ARG SHA256_S6_AMD64
 ARG SHA256_S6_ARM64
 ARG SHA256_S6_NOARCH
 
 ARG DESTDIR="/downloaded"
-ARG CHECKSUM_ALGORITHM="sha256"
+ARG S6_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${S6_CHECKSUM_ALGORITHM}"
 
 ARG S6_CHECKSUM_AMD64="${CHECKSUM_ALGORITHM}:${SHA256_S6_AMD64}"
 ARG S6_CHECKSUM_ARM64="${CHECKSUM_ALGORITHM}:${SHA256_S6_ARM64}"
@@ -144,12 +150,28 @@ ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
 ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
 ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
 
-ADD --checksum="${S6_CHECKSUM_AMD64}" "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
-ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
-ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+##ADD --checksum="${S6_CHECKSUM_AMD64}" "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
+##ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
+##ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+
+# --checksum wasn't recognized, so use busybox to check the sums instead
+ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_AMD64}" ; file="${S6_FILE_AMD64}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_ARM64}" ; file="${S6_FILE_ARM64}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_NOARCH}" ; file="${S6_FILE_NOARCH}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
 
 FROM alpine:${ALPINE_VERSION} AS s6-overlay-extracted
-COPY --link --from=s6-overlay-download /downloaded /downloaded
+COPY --from=s6-overlay-download /downloaded /downloaded
+
+ARG S6_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${S6_CHECKSUM_ALGORITHM}"
 
 ARG TARGETARCH
 
@@ -168,11 +190,12 @@ RUN set -eu ; \
       unset -v arg1 ; \
     } ; \
 \
+    apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \
     mkdir -v /verified ; \
     cd /downloaded ; \
     for f in *.sha256 ; \
     do \
-      sha256sum -c < "${f}" || exit ; \
+      "${CHECKSUM_ALGORITHM}sum" --check --warn --strict "${f}" || exit ; \
       ln -v "${f%.sha256}" /verified/ || exit ; \
     done ; \
     unset -v f ; \
@@ -191,6 +214,9 @@ RUN set -eu ; \
     set +x ; \
     unset -v f ;
 
+FROM scratch AS s6-overlay
+COPY --from=s6-overlay-extracted /s6-overlay-rootfs /
+
 FROM debian:bookworm-slim AS tubesync
 
 ARG TARGETARCH
@@ -214,8 +240,8 @@ ENV DEBIAN_FRONTEND="noninteractive" \
   S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
 
 # Install third party software
-COPY --link --from=s6-overlay-extracted /s6-overlay-rootfs /
-COPY --link --from=ffmpeg-extracted /extracted /usr/local/bin/
+COPY --from=s6-overlay / /
+COPY --from=ffmpeg /usr/local/bin/ /usr/local/bin/
 
 # Reminder: the SHELL handles all variables
 RUN set -x && \

From 45d7039188c746e9726562808caa7ed8bbc5f6ee Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 14 Jan 2025 05:34:59 -0500
Subject: [PATCH 072/143] Only log the extra messages with the new setting

---
 tubesync/sync/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index bb850af3..a65abdf8 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1174,7 +1174,8 @@ class Media(models.Model):
 
     @property
     def loaded_metadata(self):
-        self.reduce_data
+        if getattr(settings, 'SHRINK_OLD_MEDIA_METADATA', False):
+            self.reduce_data
         try:
             data = json.loads(self.metadata)
             if not isinstance(data, dict):

From ebf9ff8ebae8ef7b6eec0a0d64b6352c269d4bbf Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 15 Jan 2025 04:26:55 -0500
Subject: [PATCH 073/143] Add environment variables for container

- TUBESYNC_SHRINK_NEW
- TUBESYNC_SHRINK_OLD

These must be set to the word 'True' (case insensitive) to enable the setting.
---
 tubesync/tubesync/local_settings.py.container | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
index e75778b8..20f55098 100644
--- a/tubesync/tubesync/local_settings.py.container
+++ b/tubesync/tubesync/local_settings.py.container
@@ -87,6 +87,11 @@ SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'T
 SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False
 
 
+SHRINK_NEW_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_NEW', 'false').strip().lower()
+SHRINK_NEW_MEDIA_METADATA = ( 'true' == SHRINK_NEW_MEDIA_METADATA_STR )
+SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip().lower()
+SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR )
+
 VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))
 
 

From d349bd55c4d883353622884126db8e7060a4b298 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 15 Jan 2025 04:43:58 -0500
Subject: [PATCH 074/143] Avoid env for healthcheck

This is only intended for use in containers, so we know where python3 should be installed.

This is called very often, so we should try to use as few resources as we can.
---
 tubesync/healthcheck.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/healthcheck.py b/tubesync/healthcheck.py
index 840da640..5bc127b0 100755
--- a/tubesync/healthcheck.py
+++ b/tubesync/healthcheck.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 '''
 
     Perform an HTTP request to a URL and exit with an exit code of 1 if the

From 35f6a54823bd598b8c076ad6dd935919865d5235 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 15 Jan 2025 06:17:33 -0500
Subject: [PATCH 075/143] Balance blank lines

---
 tubesync/tubesync/local_settings.py.container | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
index 20f55098..0114e76d 100644
--- a/tubesync/tubesync/local_settings.py.container
+++ b/tubesync/tubesync/local_settings.py.container
@@ -92,6 +92,7 @@ SHRINK_NEW_MEDIA_METADATA = ( 'true' == SHRINK_NEW_MEDIA_METADATA_STR )
 SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip().lower()
 SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR )
 
+
 VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))
 
 

From 0bf72fd5f061c9c851aa4ae8f055e8d23eae68b3 Mon Sep 17 00:00:00 2001
From: FaySmash <30392780+FaySmash@users.noreply.github.com>
Date: Wed, 15 Jan 2025 18:46:05 +0100
Subject: [PATCH 076/143] Update README.md

Added a small section about potential permission issues with volumes.
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index a01f9830..e49a30c7 100644
--- a/README.md
+++ b/README.md
@@ -138,6 +138,8 @@ services:
       - PGID=1000
 ```
 
+> [!IMPORTANT]  
+> If the `/downloads` directory is mounted to a volume which points to a remote storage, make sure to suppy the `UID` and `GID` parameters in the driver options, to match the `PUID` and `PGID` specified as environment variables to prevent permission issues. [See this issue for details](https://github.com/meeb/tubesync/issues/616#issuecomment-2593458282)
 
 ## Optional authentication
 

From af0aae3de4ef85513de39201a0e6d94310a92f5f Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 01:18:20 -0500
Subject: [PATCH 077/143] Don't write 'None' in default rating

It's better to not have a rating than to create parsing problems.

An example of what is avoided:

```
<ratings>
  <rating name="youtube" max="5" default="True">
    <value>None</value>
    <votes>16781</votes>
  </rating>
</ratings>
```
---
 tubesync/sync/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 14ce4cf0..59d9e6d8 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1426,7 +1426,8 @@ class Media(models.Model):
         rating.tail = '\n  '
         ratings = nfo.makeelement('ratings', {})
         ratings.text = '\n    '
-        ratings.append(rating)
+        if self.rating is not None:
+            ratings.append(rating)
         ratings.tail = '\n  '
         nfo.append(ratings)
         # plot = media metadata description

From 57417915bf9357c170ba5748dde2d812c61dad4a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 01:27:41 -0500
Subject: [PATCH 078/143] lowercase the true value

I doubt it's unsupported by any parser, but every example uses lowercase for this, we may as well also.
---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 59d9e6d8..44a94454 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1418,7 +1418,7 @@ class Media(models.Model):
         rating_attrs = OrderedDict()
         rating_attrs['name'] = 'youtube'
         rating_attrs['max'] = '5'
-        rating_attrs['default'] = 'True'
+        rating_attrs['default'] = 'true'
         rating = nfo.makeelement('rating', rating_attrs)
         rating.text = '\n      '
         rating.append(value)

From ab7b601ad27e680e9aff58e814195563719fc2ce Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 01:31:07 -0500
Subject: [PATCH 079/143] Don't write zero into MPAA in .nfo

---
 tubesync/sync/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 14ce4cf0..546828f1 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1443,7 +1443,8 @@ class Media(models.Model):
         mpaa = nfo.makeelement('mpaa', {})
         mpaa.text = str(self.age_limit)
         mpaa.tail = '\n  '
-        nfo.append(mpaa)
+        if self.age_limit and self.age_limit > 0:
+            nfo.append(mpaa)
         # runtime = media metadata duration in seconds
         runtime = nfo.makeelement('runtime', {})
         runtime.text = str(self.duration)

From 6f00ce812061318cdd50d8808332a971d8bbf201 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 03:01:10 -0500
Subject: [PATCH 080/143] Create upgrade_yt-dlp.sh

---
 tubesync/upgrade_yt-dlp.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 tubesync/upgrade_yt-dlp.sh

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
new file mode 100644
index 00000000..e4fdd171
--- /dev/null
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+pip3() {
+    local pip_whl
+    pip_whl="$(ls -1r /usr/share/python-wheels/pip-*-py3-none-any.whl | head -n 1)"
+
+    python3 "${pip_whl}/pip" "$@"
+}
+
+pip3 install --upgrade --break-system-packages yt-dlp
+

From d7f9fa45ecb2d94fe48ea2778538fea2a961ca83 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 08:16:31 +0000
Subject: [PATCH 081/143] Add executable to sh script

---
 tubesync/upgrade_yt-dlp.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tubesync/upgrade_yt-dlp.sh

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
old mode 100644
new mode 100755

From 862c17b67656980ed852f0c88a71f67c41be5990 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 03:30:06 -0500
Subject: [PATCH 082/143] Bump ffmpeg & yt-dlp

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 554e0aaf..a83fa1da 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
 # check=error=true
 
-ARG FFMPEG_DATE="2025-01-10-19-43"
-ARG FFMPEG_VERSION="N-118280-g5cd49e1bfd"
+ARG FFMPEG_DATE="2025-01-15-14-13"
+ARG FFMPEG_VERSION="N-118315-g4f3c9f2f03"
 
 ARG S6_VERSION="3.2.0.2"
 

From 4e51d54ec122dc10550be77618250276e6a7b4b2 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 16 Jan 2025 08:24:11 -0500
Subject: [PATCH 083/143] Use pip runner from pipenv

---
 tubesync/upgrade_yt-dlp.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
index e4fdd171..21d51564 100755
--- a/tubesync/upgrade_yt-dlp.sh
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
 pip3() {
-    local pip_whl
-    pip_whl="$(ls -1r /usr/share/python-wheels/pip-*-py3-none-any.whl | head -n 1)"
+    local pip_runner pip_whl run_whl
 
-    python3 "${pip_whl}/pip" "$@"
+    # pipenv
+    pip_runner='/usr/lib/python3/dist-packages/pipenv/patched/pip/__pip-runner__.py'
+
+    # python3-pip-whl
+    pip_whl="$(ls -1r /usr/share/python-wheels/pip-*-py3-none-any.whl | head -n 1)"
+    run_whl="${pip_whl}/pip"
+
+    python3 "${pip_runner}" "$@"
 }
 
 pip3 install --upgrade --break-system-packages yt-dlp

From ccdd43845c0ae28ee803e75cd3781bb04df2113c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 17 Jan 2025 12:46:11 -0500
Subject: [PATCH 084/143] Cache for pipenv & apt

- Consolidate apk & apt commands
- Consolidate ENV layers
- Consolidate RUN layers
- Remove unused variables
- Remove packages that are no longer needed
- Bind mount /app/Pipfile

`/cache` is now a `tmpfs` mount that has a `cache` mount on top for `pipenv` to use.

`apt` has `cache` mounts in the standard places:
- /var/lib/apt
- /var/cache/apt
---
 Dockerfile | 106 +++++++++++++++++++++++++----------------------------
 1 file changed, 50 insertions(+), 56 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a83fa1da..4a366e96 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,12 +11,15 @@ ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f6
 ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
 
 ARG ALPINE_VERSION="latest"
+ARG DEBIAN_VERSION="bookworm-slim"
+
 ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION}"
 ARG FFMPEG_SUFFIX_FILE=".tar.xz"
 
 ARG FFMPEG_CHECKSUM_ALGORITHM="sha256"
 ARG S6_CHECKSUM_ALGORITHM="sha256"
 
+
 FROM alpine:${ALPINE_VERSION} AS ffmpeg-download
 ARG FFMPEG_DATE
 ARG FFMPEG_VERSION
@@ -36,7 +39,7 @@ ARG DESTDIR="/downloaded"
 ARG TARGETARCH
 ADD "${FFMPEG_URL}/${FFMPEG_FILE_SUMS}" "${DESTDIR}/"
 RUN set -eu ; \
-    apk --no-cache --no-progress add cmd:aria2c cmd:awk ; \
+    apk --no-cache --no-progress add cmd:aria2c cmd:awk "cmd:${CHECKSUM_ALGORITHM}sum" ; \
 \
     aria2c_options() { \
         algorithm="${CHECKSUM_ALGORITHM%[0-9]??}" ; \
@@ -80,8 +83,6 @@ RUN set -eu ; \
       --summary-interval=0 \
       --input-file /tmp/downloads ; \
 \
-    apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \
-\    
     decide_expected() { \
         case "${TARGETARCH}" in \
             (amd64) printf -- '%s' "${FFMPEG_CHECKSUM_AMD64}" ;; \
@@ -217,54 +218,53 @@ RUN set -eu ; \
 FROM scratch AS s6-overlay
 COPY --from=s6-overlay-extracted /s6-overlay-rootfs /
 
-FROM debian:bookworm-slim AS tubesync
-
-ARG TARGETARCH
-ARG TARGETPLATFORM
+FROM debian:${DEBIAN_VERSION} AS tubesync
 
 ARG S6_VERSION
 
 ARG FFMPEG_DATE
 ARG FFMPEG_VERSION
 
-ENV S6_VERSION="${S6_VERSION}" \
-  FFMPEG_DATE="${FFMPEG_DATE}" \
-  FFMPEG_VERSION="${FFMPEG_VERSION}"
-
 ENV DEBIAN_FRONTEND="noninteractive" \
-  HOME="/root" \
-  LANGUAGE="en_US.UTF-8" \
-  LANG="en_US.UTF-8" \
-  LC_ALL="en_US.UTF-8" \
-  TERM="xterm" \
-  S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
+    HOME="/root" \
+    LANGUAGE="en_US.UTF-8" \
+    LANG="en_US.UTF-8" \
+    LC_ALL="en_US.UTF-8" \
+    TERM="xterm" \
+    # Do not include compiled byte-code
+    PIP_NO_COMPILE=1 \
+    PIP_ROOT_USER_ACTION='ignore' \
+    S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
+
+ENV S6_VERSION="${S6_VERSION}" \
+    FFMPEG_DATE="${FFMPEG_DATE}" \
+    FFMPEG_VERSION="${FFMPEG_VERSION}"
 
 # Install third party software
 COPY --from=s6-overlay / /
 COPY --from=ffmpeg /usr/local/bin/ /usr/local/bin/
 
 # Reminder: the SHELL handles all variables
-RUN set -x && \
+RUN --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \
+    --mount=type=cache,id=apt-cache-cache,sharing=locked,target=/var/cache/apt \
+  set -x && \
+  # Update from the network and keep cache
+  rm -f /etc/apt/apt.conf.d/docker-clean && \
   apt-get update && \
+  # Install locales
   apt-get -y --no-install-recommends install locales && \
   printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \
   locale-gen en_US.UTF-8 && \
-  # Install required distro packages
-  apt-get -y --no-install-recommends install curl ca-certificates file binutils xz-utils && \
+  # Install file
+  apt-get -y --no-install-recommends install file && \
   # Installed s6 (using COPY earlier)
   file -L /command/s6-overlay-suexec && \
   # Installed ffmpeg (using COPY earlier)
   /usr/local/bin/ffmpeg -version && \
   file /usr/local/bin/ff* && \
-  # Clean up
-  apt-get -y autoremove --purge file binutils xz-utils && \
-  rm -rf /var/lib/apt/lists/* && \
-  rm -rf /var/cache/apt/* && \
-  rm -rf /tmp/*
-
-# Install dependencies we keep
-RUN set -x && \
-  apt-get update && \
+  # Clean up file
+  apt-get -y autoremove --purge file && \
+  # Install dependencies we keep
   # Install required distro packages
   apt-get -y --no-install-recommends install \
   libjpeg62-turbo \
@@ -279,29 +279,27 @@ RUN set -x && \
   redis-server \
   curl \
   less \
-  && apt-get -y autoclean && \
-  rm -rf /var/lib/apt/lists/* && \
-  rm -rf /var/cache/apt/* && \
+  && \
+  # Clean up
+  apt-get -y autopurge && \
+  apt-get -y autoclean && \
   rm -rf /tmp/*
 
 # Copy over pip.conf to use piwheels
 COPY pip.conf /etc/pip.conf
 
-# Add Pipfile
-COPY Pipfile /app/Pipfile
-
-# Do not include compiled byte-code
-ENV PIP_NO_COMPILE=1 \
-  PIP_NO_CACHE_DIR=1 \
-  PIP_ROOT_USER_ACTION='ignore'
-
 # Switch workdir to the the app
 WORKDIR /app
 
 # Set up the app
-#BuildKit#RUN --mount=type=bind,source=Pipfile,target=/app/Pipfile \
-RUN \
+RUN --mount=type=tmpfs,target=/cache \
+    --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \
+    --mount=type=cache,id=apt-lib-cache,sharing=locked,target=/var/lib/apt \
+    --mount=type=cache,id=apt-cache-cache,sharing=locked,target=/var/cache/apt \
+    --mount=type=bind,source=Pipfile,target=/app/Pipfile \
   set -x && \
+  # Update from the network and keep cache
+  rm -f /etc/apt/apt.conf.d/docker-clean && \
   apt-get update && \
   # Install required build packages
   apt-get -y --no-install-recommends install \
@@ -322,10 +320,11 @@ RUN \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
   cp -at /tmp/ "${HOME}" && \
-  PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock && \
+  HOME="/tmp/${HOME#/}" \
+  XDG_CACHE_HOME='/cache' \
+  PIPENV_VERBOSITY=64 \
+    pipenv install --system --skip-lock && \
   # Clean up
-  rm /app/Pipfile && \
-  pipenv --clear && \
   apt-get -y autoremove --purge \
   default-libmysqlclient-dev \
   g++ \
@@ -339,12 +338,9 @@ RUN \
   python3-pip \
   zlib1g-dev \
   && \
-  apt-get -y autoremove && \
+  apt-get -y autopurge && \
   apt-get -y autoclean && \
-  rm -rf /var/lib/apt/lists/* && \
-  rm -rf /var/cache/apt/* && \
-  rm -rf /tmp/*
-
+  rm -v -rf /tmp/*
 
 # Copy app
 COPY tubesync /app
@@ -362,11 +358,8 @@ RUN set -x && \
   mkdir -v -p /config/media && \
   mkdir -v -p /config/cache/pycache && \
   mkdir -v -p /downloads/audio && \
-  mkdir -v -p /downloads/video
-
-
-# Append software versions
-RUN set -x && \
+  mkdir -v -p /downloads/video && \
+  # Append software versions
   ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \
   test -n "${ffmpeg_version}" && \
   printf -- "ffmpeg_version = '%s'\n" "${ffmpeg_version}" >> /app/common/third_party_versions.py
@@ -378,7 +371,8 @@ COPY config/root /
 HEALTHCHECK --interval=1m --timeout=10s --start-period=3m CMD ["/app/healthcheck.py", "http://127.0.0.1:8080/healthcheck"]
 
 # ENVS and ports
-ENV PYTHONPATH="/app" PYTHONPYCACHEPREFIX="/config/cache/pycache"
+ENV PYTHONPATH="/app" \
+    PYTHONPYCACHEPREFIX="/config/cache/pycache"
 EXPOSE 4848
 
 # Volumes

From ae07c1ce8942385897e1ee743aaf5bf3270d1d52 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 17 Jan 2025 13:03:06 -0500
Subject: [PATCH 085/143] Fallback to run_whl

Prefer `pip_runner` but try the `pip` wheel too, if it's missing.
---
 tubesync/upgrade_yt-dlp.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
index 21d51564..c3a7edab 100755
--- a/tubesync/upgrade_yt-dlp.sh
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -5,12 +5,13 @@ pip3() {
 
     # pipenv
     pip_runner='/usr/lib/python3/dist-packages/pipenv/patched/pip/__pip-runner__.py'
+    test -s "${pip_runner}" || pip_runner=''
 
     # python3-pip-whl
     pip_whl="$(ls -1r /usr/share/python-wheels/pip-*-py3-none-any.whl | head -n 1)"
     run_whl="${pip_whl}/pip"
 
-    python3 "${pip_runner}" "$@"
+    python3 "${pip_runner:-"${run_whl}"}" "$@"
 }
 
 pip3 install --upgrade --break-system-packages yt-dlp

From eff92e3469accb46e941f75e969d86eaf89baf17 Mon Sep 17 00:00:00 2001
From: FaySmash <30392780+FaySmash@users.noreply.github.com>
Date: Fri, 17 Jan 2025 19:46:10 +0100
Subject: [PATCH 086/143] Update README.md

Revised version of the section about potential permission issues with Samba volumes.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e49a30c7..d695221f 100644
--- a/README.md
+++ b/README.md
@@ -139,7 +139,7 @@ services:
 ```
 
 > [!IMPORTANT]  
-> If the `/downloads` directory is mounted to a volume which points to a remote storage, make sure to suppy the `UID` and `GID` parameters in the driver options, to match the `PUID` and `PGID` specified as environment variables to prevent permission issues. [See this issue for details](https://github.com/meeb/tubesync/issues/616#issuecomment-2593458282)
+> If the `/downloads` directory is mounted to a [Samba volume](https://docs.docker.com/engine/storage/volumes/#create-cifssamba-volumes), make sure to suppy the `UID` and `GID` parameters in the driver options. These have to be the same as the `PUID` and `PGID`, which were specified as environment variables. This prevents issues when executing file actions (like writing metadata). [See this issue for details](https://github.com/meeb/tubesync/issues/616#issuecomment-2593458282)
 
 ## Optional authentication
 

From c5cdbe4a550fc5f38e7cd0533f7ad28e237f8390 Mon Sep 17 00:00:00 2001
From: FaySmash <30392780+FaySmash@users.noreply.github.com>
Date: Sun, 19 Jan 2025 17:18:11 +0100
Subject: [PATCH 087/143] Update README.md

Fix the suppy => supply misspelling +alternate phrasing

Co-authored-by: tcely <tcely@users.noreply.github.com>
---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d695221f..ad437bba 100644
--- a/README.md
+++ b/README.md
@@ -139,7 +139,10 @@ services:
 ```
 
 > [!IMPORTANT]  
-> If the `/downloads` directory is mounted to a [Samba volume](https://docs.docker.com/engine/storage/volumes/#create-cifssamba-volumes), make sure to suppy the `UID` and `GID` parameters in the driver options. These have to be the same as the `PUID` and `PGID`, which were specified as environment variables. This prevents issues when executing file actions (like writing metadata). [See this issue for details](https://github.com/meeb/tubesync/issues/616#issuecomment-2593458282)
+> If the `/downloads` directory is mounted from a [Samba volume](https://docs.docker.com/engine/storage/volumes/#create-cifssamba-volumes), be sure to also supply the `uid` and `gid` mount parameters in the driver options.
+> These must be matched to the `PUID` and `PGID` values, which were specified as environment variables.
+> 
+> Matching these user and group ID numbers prevents issues when executing file actions, such as writing metadata. See [this issue](https://github.com/meeb/tubesync/issues/616#issuecomment-2593458282) for details.
 
 ## Optional authentication
 

From 44db638b122d608cf3f2957dae2a4708892e9caa Mon Sep 17 00:00:00 2001
From: meeb <meeb@meeb.org>
Date: Tue, 21 Jan 2025 00:21:03 +1100
Subject: [PATCH 088/143] fix num workers comparison check, resolves #634

---
 tubesync/tubesync/gunicorn.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tubesync/tubesync/gunicorn.py b/tubesync/tubesync/gunicorn.py
index d59c1389..0058fa65 100644
--- a/tubesync/tubesync/gunicorn.py
+++ b/tubesync/tubesync/gunicorn.py
@@ -10,9 +10,10 @@ def get_num_workers():
         num_workers = int(os.getenv('GUNICORN_WORKERS', 3))
     except ValueError:
         num_workers = cpu_workers
-    if 0 > num_workers > cpu_workers:
-        num_workers = cpu_workers
-    return num_workers
+    if 0 < num_workers < cpu_workers:
+        return num_workers
+    else:
+        return cpu_workers
 
 
 def get_bind():

From 66e51929803cace51ba946eece5af1822e225d51 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 20 Jan 2025 08:49:14 -0500
Subject: [PATCH 089/143] Warn against regular updating of yt-dlp

---
 tubesync/upgrade_yt-dlp.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
index c3a7edab..b92e1fd0 100755
--- a/tubesync/upgrade_yt-dlp.sh
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
 
+warning_message() {
+    cat <<EOM
+Please report any issues that you have encountered before updating yt-dlp.
+
+This is a tool to assist developers with debugging YouTube issues.
+It should not be used as an alternative to updating container images!
+EOM
+} 1>&2
+
 pip3() {
     local pip_runner pip_whl run_whl
 
@@ -14,5 +23,8 @@ pip3() {
     python3 "${pip_runner:-"${run_whl}"}" "$@"
 }
 
+warning_message
+test -n "${TUBESYNC_DEBUG}" || exit 1
+
 pip3 install --upgrade --break-system-packages yt-dlp
 

From f65f6f1de5637b29ebee76c5ccb32388c7752c72 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 21 Jan 2025 01:12:24 -0500
Subject: [PATCH 090/143] Treat static_url the same as other URLs

---
 tubesync/tubesync/wsgi.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tubesync/tubesync/wsgi.py b/tubesync/tubesync/wsgi.py
index 71c61003..123dfde6 100644
--- a/tubesync/tubesync/wsgi.py
+++ b/tubesync/tubesync/wsgi.py
@@ -1,5 +1,4 @@
 import os
-from urllib.parse import urljoin
 from django.core.wsgi import get_wsgi_application
 
 
@@ -17,9 +16,8 @@ def application(environ, start_response):
             raise Exception(f'DJANGO_URL_PREFIX must end with a /, '
                             f'got: {DJANGO_URL_PREFIX}')
     if script_name:
-        static_url = urljoin(script_name, 'static/')
         environ['SCRIPT_NAME'] = script_name
         path_info = environ['PATH_INFO']
-        if path_info.startswith(script_name) and not path_info.startswith(static_url):
+        if path_info.startswith(script_name):
             environ['PATH_INFO'] = path_info[len(script_name) - 1:]
     return _application(environ, start_response)

From 52d703ff1ff358f9ae4a44d54b33601160a82737 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 21 Jan 2025 01:55:13 -0500
Subject: [PATCH 091/143] Better check for script_name

---
 tubesync/tubesync/wsgi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/tubesync/wsgi.py b/tubesync/tubesync/wsgi.py
index 123dfde6..74912aef 100644
--- a/tubesync/tubesync/wsgi.py
+++ b/tubesync/tubesync/wsgi.py
@@ -15,7 +15,7 @@ def application(environ, start_response):
         else:
             raise Exception(f'DJANGO_URL_PREFIX must end with a /, '
                             f'got: {DJANGO_URL_PREFIX}')
-    if script_name:
+    if script_name is not None:
         environ['SCRIPT_NAME'] = script_name
         path_info = environ['PATH_INFO']
         if path_info.startswith(script_name):

From 1f95b858f2ef7ec6cc9c2994327474be8c4cbd5f Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 21 Jan 2025 03:08:24 -0500
Subject: [PATCH 092/143] Use --break-system-packages with pip

Unfortunately, both versions of `pip` don't have this flag.

Check the version, then add the flag if it is not too old.
---
 tubesync/upgrade_yt-dlp.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
index b92e1fd0..9da6d555 100755
--- a/tubesync/upgrade_yt-dlp.sh
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -26,5 +26,13 @@ pip3() {
 warning_message
 test -n "${TUBESYNC_DEBUG}" || exit 1
 
-pip3 install --upgrade --break-system-packages yt-dlp
+# Use the flag added in 23.0.1, if possible.
+# https://github.com/pypa/pip/pull/11780
+break_system_packages='--break-system-packages'
+pip_version="$(pip3 --version | awk '$1 = "pip" { print $2; exit; }')"
+if [[ "${pip_version}" < "23.0.1" ]]; then
+    break_system_packages=''
+fi
+
+pip3 install --upgrade ${break_system_packages} yt-dlp
 

From 0ea508443ade4ae05b9ea19500c3d7f1dc1a8b93 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 21 Jan 2025 04:21:47 -0500
Subject: [PATCH 093/143] Pipefile -> Pipfile

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ad437bba..dbf4b6fe 100644
--- a/README.md
+++ b/README.md
@@ -325,7 +325,7 @@ Notable libraries and software used:
  * [django-sass](https://github.com/coderedcorp/django-sass/)
  * The container bundles with `s6-init` and `nginx`
 
-See the [Pipefile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a full list.
+See the [Pipfile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a full list.
 
 ### Can I get access to the full Django admin?
 

From 96078f8d40cfd3b4f75e3d51ac1667cefad78aad Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 21 Jan 2025 04:38:06 -0500
Subject: [PATCH 094/143] Update architectures FAQ

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ad437bba..ce89cb7a 100644
--- a/README.md
+++ b/README.md
@@ -353,7 +353,12 @@ etc.). Configuration of this is beyond the scope of this README.
 
 ### What architectures does the container support?
 
-Just `amd64` for the moment. Others may be made available if there is demand.
+Only two are supported, for the moment:
+- `amd64` (most desktop PCs and servers)
+-  `arm64`
+(modern ARM computers, such as the Rasperry Pi 3 or later)
+
+Others may be made available, if there is demand.
 
 ### The pipenv install fails with "Locking failed"!
 

From 1671c6e7066e83d8ee684c9256726fbcada329ee Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 04:55:57 -0500
Subject: [PATCH 095/143] DRY YouTube domain list

I am tired of links copied from YouTube not working without adjustments.
---
 tubesync/sync/views.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 52090042..8ca853de 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -193,10 +193,15 @@ class ValidateSourceView(FormView):
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
                                               'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r')
     }
+    _youtube_domains = frozenset({
+        'youtube.com',
+        'm.youtube.com',
+        'www.youtube.com',
+    })
     validation_urls = {
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domains': _youtube_domains,
             'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -205,7 +210,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domains': _youtube_domains,
             'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -214,7 +219,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domains': _youtube_domains,
             'path_regex': '^\/(playlist|watch)$',
             'path_must_not_match': (),
             'qs_args': ('list',),

From b38c7d7c7f03598c4450d71433292ff92c9f7eb5 Mon Sep 17 00:00:00 2001
From: meeb <meeb@meeb.org>
Date: Wed, 22 Jan 2025 23:59:41 +1100
Subject: [PATCH 096/143] bump ffmpeg and yt-dlp

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4a366e96..023f4fd8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
 # check=error=true
 
-ARG FFMPEG_DATE="2025-01-15-14-13"
-ARG FFMPEG_VERSION="N-118315-g4f3c9f2f03"
+ARG FFMPEG_DATE="2025-01-21-14-19"
+ARG FFMPEG_VERSION="N-118328-g504df09c34"
 
 ARG S6_VERSION="3.2.0.2"
 

From ba321945b6239f99403e429c6ff268c9de336537 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 19:41:03 -0500
Subject: [PATCH 097/143] Automated channel_id extraction

---
 tubesync/sync/views.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 8ca853de..ff4a87f4 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -291,11 +291,24 @@ class ValidateSourceView(FormView):
         url = reverse_lazy('sync:add-source')
         fields_to_populate = self.prepopulate_fields.get(self.source_type)
         fields = {}
+        value = self.key
+        use_channel_id = (
+            'youtube-channel' == self.source_type_str and
+            '@' == self.key[0]
+        )
+        if use_channel_id:
+            self.source_type_str = 'youtube-channel-id'
+            self.source_type = self.source_types.get(self.source_type_str, None)
+            self.key = youtube.get_channel_id(
+                Source.create_index_url(self.source_type, self.key, 'videos')
+            )
         for field in fields_to_populate:
             if field == 'source_type':
                 fields[field] = self.source_type
-            elif field in ('key', 'name', 'directory'):
+            elif field == 'key':
                 fields[field] = self.key
+            elif field in ('name', 'directory'):
+                fields[field] = value
         return append_uri_params(url, fields)
 
 

From e5b4e9dbc0eb418827f3fac308dcd8b58118911c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 20:07:59 -0500
Subject: [PATCH 098/143] Add get_channel_id

---
 tubesync/sync/youtube.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 5fdef3cb..5371c937 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -45,6 +45,31 @@ def get_yt_opts():
         opts.update({'cookiefile': cookie_file_path})
     return opts
 
+def get_channel_id(url):
+    # yt-dlp --simulate --no-check-formats --playlist-items 1
+    #   --print 'pre_process:%(playlist_channel_id,playlist_id,channel_id)s'
+    opts = get_yt_opts()
+    opts.update({
+        'skip_download': True,
+        'simulate': True,
+        'logger': log,
+        'extract_flat': True,  # Change to False to get detailed info
+        'check_formats': False,
+        'playlist_items': '1',
+    })
+
+    with yt_dlp.YoutubeDL(opts) as y:
+        try:
+            response = y.extract_info(url, download=False)
+            
+            channel_id = response['channel_id']
+            playlist_id = response['playlist_id']
+            playlist_channel_id = response['playlist_channel_id']
+        except yt_dlp.utils.DownloadError as e:
+            raise YouTubeError(f'Failed to extract channel ID for "{url}": {e}') from e
+        else:
+            return playlist_channel_id or playlist_id or channel_id
+
 def get_channel_image_info(url):
     opts = get_yt_opts()
     opts.update({

From 5546c5dad24aaae939392de851024ff298edbd42 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 20:21:23 -0500
Subject: [PATCH 099/143] Use the channel type to fetch channel_id

---
 tubesync/sync/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index ff4a87f4..17f88522 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -297,10 +297,11 @@ class ValidateSourceView(FormView):
             '@' == self.key[0]
         )
         if use_channel_id:
+            source_type = self.source_type
             self.source_type_str = 'youtube-channel-id'
             self.source_type = self.source_types.get(self.source_type_str, None)
             self.key = youtube.get_channel_id(
-                Source.create_index_url(self.source_type, self.key, 'videos')
+                Source.create_index_url(source_type, self.key, 'videos')
             )
         for field in fields_to_populate:
             if field == 'source_type':

From 25892388a1316b67dd16e38cc1fd31c50651b1e6 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 20:42:18 -0500
Subject: [PATCH 100/143] Remove the /channel/ from the URL

---
 tubesync/sync/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 17f88522..fe49f105 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -297,11 +297,11 @@ class ValidateSourceView(FormView):
             '@' == self.key[0]
         )
         if use_channel_id:
-            source_type = self.source_type
             self.source_type_str = 'youtube-channel-id'
             self.source_type = self.source_types.get(self.source_type_str, None)
+            url = Source.create_index_url(self.source_type, self.key, 'videos')
             self.key = youtube.get_channel_id(
-                Source.create_index_url(source_type, self.key, 'videos')
+                url.replace('/channel/', '/')
             )
         for field in fields_to_populate:
             if field == 'source_type':

From fb87b54300bd590f7a703a4b10788cdf7ef34d39 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 20:59:21 -0500
Subject: [PATCH 101/143] channel_id is the only available key

---
 tubesync/sync/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 5371c937..27dc88fc 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -61,14 +61,15 @@ def get_channel_id(url):
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             response = y.extract_info(url, download=False)
-            
-            channel_id = response['channel_id']
-            playlist_id = response['playlist_id']
-            playlist_channel_id = response['playlist_channel_id']
         except yt_dlp.utils.DownloadError as e:
             raise YouTubeError(f'Failed to extract channel ID for "{url}": {e}') from e
         else:
-            return playlist_channel_id or playlist_id or channel_id
+            try:
+                channel_id = response['channel_id']
+            except Exception as e:
+                raise YouTubeError(f'Failed to extract channel ID for "{url}": {e}') from e
+            else:
+                return channel_id
 
 def get_channel_image_info(url):
     opts = get_yt_opts()

From ed381715b5840c8976f62b63020e5e480b3c3e29 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 22 Jan 2025 21:09:03 -0500
Subject: [PATCH 102/143] Fail to previous behavior

---
 tubesync/sync/views.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index fe49f105..dccf1820 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -297,12 +297,23 @@ class ValidateSourceView(FormView):
             '@' == self.key[0]
         )
         if use_channel_id:
+            old_key = self.key
+            old_source_type = self.source_type
+            old_source_type_str = self.source_type_str
+
             self.source_type_str = 'youtube-channel-id'
             self.source_type = self.source_types.get(self.source_type_str, None)
-            url = Source.create_index_url(self.source_type, self.key, 'videos')
-            self.key = youtube.get_channel_id(
-                url.replace('/channel/', '/')
-            )
+            index_url = Source.create_index_url(self.source_type, self.key, 'videos')
+            try:
+                self.key = youtube.get_channel_id(
+                    index_url.replace('/channel/', '/')
+                )
+            except youtube.YouTubeError as e:
+                # It did not work, revert to previous behavior
+                self.key = old_key
+                self.source_type = old_source_type
+                self.source_type_str = old_source_type_str
+
         for field in fields_to_populate:
             if field == 'source_type':
                 fields[field] = self.source_type

From 3d148fc5aafb47c95e07a228eeb33ceb656e299a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 24 Jan 2025 08:22:46 -0500
Subject: [PATCH 103/143] Return `format_note` also

---
 tubesync/sync/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 1d67af38..5d2c6921 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -304,6 +304,7 @@ def parse_media_format(format_dict):
     return {
         'id': format_dict.get('format_id', ''),
         'format': format_str,
+        'format_note': format_dict.get('format_note', ''),
         'format_verbose': format_dict.get('format', ''),
         'height': height,
         'width': width,

From d4a5a78831bf2cb27236a7d39fedd3e730343736 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 24 Jan 2025 08:26:19 -0500
Subject: [PATCH 104/143] Display `format_note` after audio-only formats

---
 tubesync/sync/templates/sync/media-item.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 6f751be6..026e5a54 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -146,7 +146,7 @@
           <div>
             ID: <strong>{{ format.format_id }}</strong>
             {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %}
-            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% endif %}
+            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz {{ format.format_note }}{% endif %}
             {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
           </div>
           {% empty %}

From ee303c638b066cc74669214ba235e6874eb2eaa2 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 24 Jan 2025 09:14:53 -0500
Subject: [PATCH 105/143] Display database_filesize

---
 tubesync/sync/templates/sync/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/templates/sync/dashboard.html b/tubesync/sync/templates/sync/dashboard.html
index 8c27684c..4d9fc2da 100644
--- a/tubesync/sync/templates/sync/dashboard.html
+++ b/tubesync/sync/templates/sync/dashboard.html
@@ -125,7 +125,7 @@
       </tr>
       <tr title="Database connection used by TubeSync">
         <td class="hide-on-small-only">Database</td>
-        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
+        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }} {{ database_filesize|filesizeformat }}</strong></td>
       </tr>
     </table>
   </div>

From b65ecc43ffc7ad40c68fe171fc9c9df42c93a534 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 24 Jan 2025 09:20:12 -0500
Subject: [PATCH 106/143] Pass raw bytes count as `database_filesize`

---
 tubesync/sync/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 3fb23044..7f77e858 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -86,11 +86,11 @@ class DashboardView(TemplateView):
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
         data['database_connection'] = settings.DATABASE_CONNECTION_STR
         # Add the database filesize when using db.sqlite3
+        data['database_filesize'] = None
         db_name = str(connection.get_connection_params()['database'])
         db_path = pathlib.Path(db_name) if '/' == db_name[0] else None
         if db_path and 'sqlite' == connection.vendor:
-            db_size = db_path.stat().st_size
-            data['database_connection'] += f' ({db_size:,} bytes)'
+            data['database_filesize'] = db_path.stat().st_size
         return data
 
 

From e16f6bb86a723760b9ccd10815bfc432f82fde0a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 24 Jan 2025 09:26:04 -0500
Subject: [PATCH 107/143] Display `database_filesize` only if set

---
 tubesync/sync/templates/sync/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/templates/sync/dashboard.html b/tubesync/sync/templates/sync/dashboard.html
index 4d9fc2da..ccf4a6c3 100644
--- a/tubesync/sync/templates/sync/dashboard.html
+++ b/tubesync/sync/templates/sync/dashboard.html
@@ -125,7 +125,7 @@
       </tr>
       <tr title="Database connection used by TubeSync">
         <td class="hide-on-small-only">Database</td>
-        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }} {{ database_filesize|filesizeformat }}</strong></td>
+        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}{% if database_filesize %} {{ database_filesize|filesizeformat }}{% endif %}</strong></td>
       </tr>
     </table>
   </div>

From 56e1aae5cd2412acffa5a1c21bf5b5fea0b8621b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 09:46:03 -0500
Subject: [PATCH 108/143] Simplify `get_queryset`

---
 tubesync/sync/views.py | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 43a51b91..53be2620 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -524,20 +524,15 @@ class MediaView(ListView):
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
+        q = Media.objects.all()
+
         if self.filter_source:
-            if self.show_skipped:
-                q = Media.objects.filter(source=self.filter_source)
-            elif self.only_skipped:
-                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
-            else:
-                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
-        else:
-            if self.show_skipped:
-                q = Media.objects.all()
-            elif self.only_skipped:
-                q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
-            else:
-                q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
+            q = q.filter(source=self.filter_source)
+        if self.only_skipped:
+            q = q.filter(Q(can_download=False) | Q(skip=True) | Q(manual_skip=True))
+        elif not self.show_skipped:
+            q = q.filter(Q(can_download=True) & Q(skip=False) & Q(manual_skip=False))
+
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):

From 3b20f07450f51fbb7caf6c3b3fab77b5abc3d265 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 19:12:36 -0500
Subject: [PATCH 109/143] Try to download thumbnail before copy

---
 tubesync/sync/tasks.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index ab92e2c8..f0a8856d 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -442,10 +442,14 @@ def download_media(media_id):
                 media.downloaded_format = 'audio'
         media.save()
         # If selected, copy the thumbnail over as well
-        if media.source.copy_thumbnails and media.thumb:
-            log.info(f'Copying media thumbnail from: {media.thumb.path} '
-                     f'to: {media.thumbpath}')
-            copyfile(media.thumb.path, media.thumbpath)
+        if media.source.copy_thumbnails:
+            if not media.thumb_file_exists:
+                if download_media_thumbnail.now(str(media.pk), media.thumbnail):
+                    media.refresh_from_db()
+            if media.thumb_file_exists:
+                log.info(f'Copying media thumbnail from: {media.thumb.path} '
+                         f'to: {media.thumbpath}')
+                copyfile(media.thumb.path, media.thumbpath)
         # If selected, write an NFO file
         if media.source.write_nfo:
             log.info(f'Writing media NFO file to: {media.nfopath}')

From fe39717985a8d95f8a4dcb01ab63e91891e2f5b5 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 19:23:52 -0500
Subject: [PATCH 110/143] Change media download priority

---
 tubesync/sync/signals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index cd8cf621..e2a1398c 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -174,7 +174,7 @@ def media_post_save(sender, instance, created, **kwargs):
         download_media(
             str(instance.pk),
             queue=str(instance.source.pk),
-            priority=15,
+            priority=10,
             verbose_name=verbose_name.format(instance.name),
             remove_existing_tasks=True
         )

From 69e895818ee5b3fe973ca21de65625234697b751 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 19:46:58 -0500
Subject: [PATCH 111/143] Drop the thumbnail task if media downloaded first

---
 tubesync/sync/tasks.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index f0a8856d..8d52b3bf 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -444,8 +444,12 @@ def download_media(media_id):
         # If selected, copy the thumbnail over as well
         if media.source.copy_thumbnails:
             if not media.thumb_file_exists:
-                if download_media_thumbnail.now(str(media.pk), media.thumbnail):
-                    media.refresh_from_db()
+                thumbnail_url = media.thumbnail
+                if thumbnail_url:
+                    args = ( str(media.pk), thumbnail_url, )
+                    delete_task_by_media('sync.tasks.download_media_thumbnail', args)
+                    if download_media_thumbnail.now(*args):
+                        media.refresh_from_db()
             if media.thumb_file_exists:
                 log.info(f'Copying media thumbnail from: {media.thumb.path} '
                          f'to: {media.thumbpath}')

From b319101c703a15094fefac025b11250ca5894133 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 20:08:10 -0500
Subject: [PATCH 112/143] Remove unnecessary spaces

---
 tubesync/sync/models.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index b5d7be06..c06ba021 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -537,7 +537,7 @@ class Source(models.Model):
     def get_image_url(self):
         if self.source_type == self.SOURCE_TYPE_YOUTUBE_PLAYLIST:
             raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.')
-        
+
         return get_youtube_channel_image_info(self.url)
 
 
@@ -967,7 +967,7 @@ class Media(models.Model):
 
     def get_best_video_format(self):
         return get_best_video_format(self)
-    
+
     def get_format_str(self):
         '''
             Returns a youtube-dl compatible format string for the best matches
@@ -992,7 +992,7 @@ class Media(models.Model):
                 else:
                     return False
         return False
-    
+ 
     def get_display_format(self, format_str):
         '''
             Returns a tuple used in the format component of the output filename. This
@@ -1155,7 +1155,7 @@ class Media(models.Model):
             old_mdl = len(self.metadata or "")
             data = json.loads(self.metadata or "")
             compact_json = json.dumps(data, separators=(',', ':'), default=json_serial)
-            
+
             filtered_data = filter_response(data, True)
             filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial)
         except Exception as e:
@@ -1323,7 +1323,7 @@ class Media(models.Model):
             filename = self.filename
         prefix, ext = os.path.splitext(os.path.basename(filename))
         return f'{prefix}.nfo'
-    
+
     @property
     def nfopath(self):
         return self.directory_path / self.nfoname
@@ -1336,7 +1336,7 @@ class Media(models.Model):
             filename = self.filename
         prefix, ext = os.path.splitext(os.path.basename(filename))
         return f'{prefix}.info.json'
-    
+
     @property
     def jsonpath(self):
         return self.directory_path / self.jsonname
@@ -1564,7 +1564,7 @@ class Media(models.Model):
 
         if use_padding:
             return f'{episode_number:02}'
-        
+
         return str(episode_number)
 
 

From de1490c416116ab43e3c2b6f608659b34d1e412b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 20:11:19 -0500
Subject: [PATCH 113/143] Remove unnecessary spaces

---
 tubesync/sync/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py
index b96f4cd2..2514c525 100644
--- a/tubesync/sync/fields.py
+++ b/tubesync/sync/fields.py
@@ -60,7 +60,7 @@ class CommaSepChoiceField(models.Field):
 
         for t in self.possible_choices:
             choiceArray.append(t)
-        
+
         return choiceArray
 
     def formfield(self, **kwargs):

From 32fad9140143a45925a538e035b17a79c14281ce Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 20:12:55 -0500
Subject: [PATCH 114/143] Remove unnecessary spaces

---
 tubesync/sync/urls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py
index 66558379..116beca5 100644
--- a/tubesync/sync/urls.py
+++ b/tubesync/sync/urls.py
@@ -17,7 +17,7 @@ urlpatterns = [
     path('',
          DashboardView.as_view(),
          name='dashboard'),
-    
+
     # Source URLs
 
     path('sources',

From 7169ba5865d697ef39c7a201ebf4c6198e3018af Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 20:14:46 -0500
Subject: [PATCH 115/143] Remove unnecessary spaces

---
 tubesync/sync/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 27dc88fc..e2138f9a 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -83,7 +83,7 @@ def get_channel_image_info(url):
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             response = y.extract_info(url, download=False)
-            
+
             avatar_url = None
             banner_url = None
             for thumbnail in response['thumbnails']:
@@ -93,7 +93,7 @@ def get_channel_image_info(url):
                     banner_url = thumbnail['url']
                 if banner_url != None and avatar_url != None:
                     break
-                    
+
             return avatar_url, banner_url
         except yt_dlp.utils.DownloadError as e:
             raise YouTubeError(f'Failed to extract channel info for "{url}": {e}') from e

From 31e25eb22b8e347b937d9b616e599815db79457f Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 20:21:59 -0500
Subject: [PATCH 116/143] Remove unnecessary spaces

---
 tubesync/sync/views.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 43a51b91..898db969 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -112,7 +112,7 @@ class SourcesView(ListView):
             sobj = Source.objects.get(pk=kwargs["pk"])
             if sobj is None:
                 return HttpResponseNotFound()
-            
+
             verbose_name = _('Index media from source "{}" once')
             index_source_task(
                 str(sobj.pk),
@@ -354,7 +354,7 @@ class EditSourceMixin:
         obj = form.save(commit=False)
         source_type = form.cleaned_data['media_format']
         example_media_file = obj.get_example_media_format()
-        
+
         if example_media_file == '':
             form.add_error(
                 'media_format',
@@ -776,18 +776,18 @@ class MediaContent(DetailView):
                 pth = pth[1]
             else:
                 pth = pth[0]
-            
-            
+
+
             # build final path
             filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
-            
+
             if filepth.exists():
                 # return file
                 response = FileResponse(open(filepth,'rb'))
                 return response
             else:
                 return HttpResponseNotFound()
-            
+
         else:
             headers = {
                 'Content-Type': self.object.content_type,

From fe4f6dd6b885a374381d8e6ec8b62419964dd5e2 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sat, 25 Jan 2025 22:26:39 -0500
Subject: [PATCH 117/143] DRY {JSON,nfo,thumb}name functions

---
 tubesync/sync/models.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index b5d7be06..9008a708 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1302,13 +1302,18 @@ class Media(models.Model):
     def filepath(self):
         return self.source.directory_path / self.filename
 
-    @property
-    def thumbname(self):
+    def filename_prefix(self):
         if self.downloaded and self.media_file:
-            filename = self.media_file.path
+            mf_path = Path(self.media_file.path)
+            filename = str(mf_path.relative_to(self.source.directory_path))
         else:
             filename = self.filename
-        prefix, ext = os.path.splitext(os.path.basename(filename))
+        prefix, ext = os.path.splitext(filename)
+        return prefix
+
+    @property
+    def thumbname(self):
+        prefix = self.filename_prefix()
         return f'{prefix}.jpg'
 
     @property
@@ -1317,11 +1322,7 @@ class Media(models.Model):
 
     @property
     def nfoname(self):
-        if self.downloaded and self.media_file:
-            filename = self.media_file.path
-        else:
-            filename = self.filename
-        prefix, ext = os.path.splitext(os.path.basename(filename))
+        prefix = self.filename_prefix()
         return f'{prefix}.nfo'
     
     @property
@@ -1330,11 +1331,7 @@ class Media(models.Model):
 
     @property
     def jsonname(self):
-        if self.downloaded and self.media_file:
-            filename = self.media_file.path
-        else:
-            filename = self.filename
-        prefix, ext = os.path.splitext(os.path.basename(filename))
+        prefix = self.filename_prefix()
         return f'{prefix}.info.json'
     
     @property

From 7b1aedf7aea006c98acf0fe75385221e93cefe8c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 26 Jan 2025 07:11:30 -0500
Subject: [PATCH 118/143] Put os.path.basename() back

Without this the subdirectory from `media_format` is doubled.
---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 9008a708..72ea2cd8 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1308,7 +1308,7 @@ class Media(models.Model):
             filename = str(mf_path.relative_to(self.source.directory_path))
         else:
             filename = self.filename
-        prefix, ext = os.path.splitext(filename)
+        prefix, ext = os.path.splitext(os.path.basename(filename))
         return prefix
 
     @property

From bcbafaf8422084a5f7c58d35645547f097dff65b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 26 Jan 2025 07:54:14 -0500
Subject: [PATCH 119/143] Remove directory manipulation

Added a comment about why these directories aren't matching.
---
 tubesync/sync/models.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 72ea2cd8..cfb6ca38 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1304,10 +1304,12 @@ class Media(models.Model):
 
     def filename_prefix(self):
         if self.downloaded and self.media_file:
-            mf_path = Path(self.media_file.path)
-            filename = str(mf_path.relative_to(self.source.directory_path))
+            filename = self.media_file.path
         else:
             filename = self.filename
+        # The returned prefix should not contain any directories.
+        # So, we do not care about the different directories
+        # used for filename in the cases above.
         prefix, ext = os.path.splitext(os.path.basename(filename))
         return prefix
 

From 09ecbf210e75557c9d7240ea2dd2c2e17f53c7e5 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 02:57:47 -0500
Subject: [PATCH 120/143] Default to valid JSON

This prevents useless logged errors when there is no metadata available yet.
---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index b5d7be06..fa1ce1b4 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1153,7 +1153,7 @@ class Media(models.Model):
             from common.utils import json_serial
 
             old_mdl = len(self.metadata or "")
-            data = json.loads(self.metadata or "")
+            data = json.loads(self.metadata or "{}")
             compact_json = json.dumps(data, separators=(',', ':'), default=json_serial)
             
             filtered_data = filter_response(data, True)

From 9fc773b5c0dcb5dc2387e83b05e447efed2767d4 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 03:43:56 -0500
Subject: [PATCH 121/143] Add language_code from response

---
 tubesync/sync/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 5d2c6921..b5be2f51 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -301,11 +301,13 @@ def parse_media_format(format_dict):
             format_str = f'{height}P'
         else:
             format_str = None
+    language = format_dict.get('language', None)
     return {
         'id': format_dict.get('format_id', ''),
         'format': format_str,
         'format_note': format_dict.get('format_note', ''),
         'format_verbose': format_dict.get('format', ''),
+        'language_code': language,
         'height': height,
         'width': width,
         'vcodec': vcodec,

From 8a3a7aa333d9b5f16c692229c91dadd2f7769ec7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 03:50:42 -0500
Subject: [PATCH 122/143] Display language_code for audio formats

---
 tubesync/sync/templates/sync/media-item.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 026e5a54..90ce9ec8 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -146,7 +146,7 @@
           <div>
             ID: <strong>{{ format.format_id }}</strong>
             {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %}
-            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz {{ format.format_note }}{% endif %}
+            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% if format.language_code %} [{{ format.language_code }}]{% endif %} {{ format.format_note }}{% endif %}
             {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
           </div>
           {% empty %}

From d160a043e5a4d03a0f92218a4718a2283a16f1f7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 12:45:32 -0500
Subject: [PATCH 123/143] Use yt_dlp.get_postprocessors()

---
 tubesync/sync/youtube.py | 51 +++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 19 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 27dc88fc..1550798c 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -5,10 +5,13 @@
 
 
 import os
-from pathlib import Path
-from django.conf import settings
-from copy import copy
+
+from collections import namedtuple
 from common.logger import log
+from copy import copy, deepcopy
+from pathlib import Path
+
+from django.conf import settings
 import yt_dlp
 
 
@@ -207,6 +210,21 @@ def download_media(url, media_format, extension, output_file, info_json,
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
 
     hook.download_progress = 0
+
+    default_opts = yt_dlp.parse_options([]).options
+    pp_opts = deepcopy(default_opts.__dict__)
+    pp_opts.update({
+        'embedthumbnail': embed_thumbnail,
+        'addmetadata': embed_metadata,
+        'addchapters': True,
+        'embed_infojson': False,
+        'force_keyframes_at_cuts': True,
+    })
+
+    if skip_sponsors:
+        pp_opts['sponsorblock_mark'].update('all,-chapter'.split(','))
+        pp_opts['sponsorblock_remove'].update(sponsor_categories or {})
+
     ytopts = {
         'format': media_format,
         'merge_output_format': extension,
@@ -221,27 +239,22 @@ def download_media(url, media_format, extension, output_file, info_json,
         'writeautomaticsub': auto_subtitles,
         'subtitleslangs': sub_langs.split(','),
     }
-    if not sponsor_categories:
-        sponsor_categories = []
-    sbopt = {
-        'key': 'SponsorBlock',
-        'categories': sponsor_categories
-    }
-    ffmdopt = {
-        'key': 'FFmpegMetadata',
-        'add_chapters': embed_metadata,
-        'add_metadata': embed_metadata
-    }
     opts = get_yt_opts()
     ytopts['paths'] = opts.get('paths', {})
     ytopts['paths'].update({
         'home': os.path.dirname(output_file),
     })
-    if embed_thumbnail:
-        ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
-    if skip_sponsors:
-        ytopts['postprocessors'].append(sbopt)
-    ytopts['postprocessors'].append(ffmdopt)
+
+    # clean-up incompatible keys
+    pp_opts = {k: v for k, v in pp_opts.items() if not k.startswith('_')}
+
+    # convert dict to namedtuple
+    yt_dlp_opts = namedtuple('yt_dlp_opts', pp_opts)
+    pp_opts = yt_dlp_opts(**pp_opts)
+
+    # create the post processors list
+    ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts))
+
     opts.update(ytopts)
 
     with yt_dlp.YoutubeDL(opts) as y:

From eb27bf15832ba1e1e33f2a5c476fd76e529edb85 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 13:17:16 -0500
Subject: [PATCH 124/143] Only save() the current Media

---
 tubesync/sync/models.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 18edccfd..19a5212d 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -1600,8 +1600,7 @@ class Media(models.Model):
 
                     # update the media_file in the db
                     self.media_file.name = str(new_video_path.relative_to(self.media_file.storage.location))
-                    self.save(update_fields={'media_file'})
-                    self.refresh_from_db(fields={'media_file'})
+                    self.save()
                     log.info(f'Updated "media_file" in the database for: {self!s}')
 
                     (new_prefix_path, new_stem) = directory_and_stem(new_video_path)

From ec16e4c54df87a4ba6c71dacb9f2a698fa02a887 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:31:16 -0500
Subject: [PATCH 125/143] Use the parsed formats

---
 tubesync/sync/templates/sync/media-item.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 90ce9ec8..8e5b6fcc 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -142,12 +142,12 @@
       <tr title="The available media formats">
         <td class="hide-on-small-only">Available formats</td>
         <td><span class="hide-on-med-and-up">Available formats<br></span>
-          {% for format in media.formats %}
+          {% for format in media.iter_formats %}
           <div>
-            ID: <strong>{{ format.format_id }}</strong>
-            {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %}
+            ID: <strong>{{ format.id }}</strong>
+            {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.vbr }}k{% endif %}
             {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% if format.language_code %} [{{ format.language_code }}]{% endif %} {{ format.format_note }}{% endif %}
-            {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
+            {% if format.id == combined_format or format.id == audio_format or format.id == video_format %}<strong>(matched)</strong>{% endif %}
           </div>
           {% empty %}
           Media has no indexed available formats

From d2641c39cd4ed555da36918a1ccc21ad0864676a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:34:10 -0500
Subject: [PATCH 126/143] Add `asr` to parsed formats

---
 tubesync/sync/utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index b5be2f51..f0bef5c5 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -301,13 +301,13 @@ def parse_media_format(format_dict):
             format_str = f'{height}P'
         else:
             format_str = None
-    language = format_dict.get('language', None)
+
     return {
         'id': format_dict.get('format_id', ''),
         'format': format_str,
         'format_note': format_dict.get('format_note', ''),
         'format_verbose': format_dict.get('format', ''),
-        'language_code': language,
+        'language_code': format_dict.get('language', None),
         'height': height,
         'width': width,
         'vcodec': vcodec,
@@ -315,6 +315,7 @@ def parse_media_format(format_dict):
         'vbr': format_dict.get('tbr', 0),
         'acodec': acodec,
         'abr': format_dict.get('abr', 0),
+        'asr': format_dict.get('asr', 0),
         'is_60fps': fps > 50,
         'is_hdr': 'HDR' in format_dict.get('format', '').upper(),
         'is_hls': is_hls,

From bb226d4bc103f10dd906c26fe7d46fbf43ca8996 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 15:08:24 -0500
Subject: [PATCH 127/143] Made the output more friendly

- Humanize the Hz number
- Do not output '@Nonek' for combined audio / video
---
 tubesync/sync/templates/sync/media-item.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 8e5b6fcc..3af853cc 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -1,4 +1,4 @@
-{% extends 'base.html' %}{% load static %}
+{% extends 'base.html' %}{% load static %}{% load humanize %}
 
 {% block headtitle %}Media - {{ media.key }}{% endblock %}
 
@@ -146,7 +146,7 @@
           <div>
             ID: <strong>{{ format.id }}</strong>
             {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.vbr }}k{% endif %}
-            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% if format.language_code %} [{{ format.language_code }}]{% endif %} {{ format.format_note }}{% endif %}
+            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} {% if format.abr %}@{{ format.abr }}k / {% endif %}{{ format.asr|intcomma }}Hz{% if format.language_code and format.abr %} [{{ format.language_code }}]{% endif %} {{ format.format_note }}{% endif %}
             {% if format.id == combined_format or format.id == audio_format or format.id == video_format %}<strong>(matched)</strong>{% endif %}
           </div>
           {% empty %}

From 133f08bd867a2fb79313006684f22394cc101250 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 15:17:39 -0500
Subject: [PATCH 128/143] Hide format_note for combined audio / video

---
 tubesync/sync/templates/sync/media-item.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 3af853cc..d54f4efd 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -146,7 +146,7 @@
           <div>
             ID: <strong>{{ format.id }}</strong>
             {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.vbr }}k{% endif %}
-            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} {% if format.abr %}@{{ format.abr }}k / {% endif %}{{ format.asr|intcomma }}Hz{% if format.language_code and format.abr %} [{{ format.language_code }}]{% endif %} {{ format.format_note }}{% endif %}
+            {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} {% if format.abr %}@{{ format.abr }}k / {% endif %}{{ format.asr|intcomma }}Hz{% if format.language_code %} [{{ format.language_code }}]{% endif %}{% if format.abr %} {{ format.format_note }}{% endif %}{% endif %}
             {% if format.id == combined_format or format.id == audio_format or format.id == video_format %}<strong>(matched)</strong>{% endif %}
           </div>
           {% empty %}

From d34e1a204e67ffd618a550c27c215c046caa0029 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 16:12:52 -0500
Subject: [PATCH 129/143] Set a default nice level

Now that SponsorBlock is running, these jobs are using significant CPU resources.

Setting a nice level to let other things run better while `ffmpeg` is running seems wise.
---
 config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run
index da22666b..b2c3a841 100755
--- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run
@@ -1,4 +1,4 @@
 #!/command/with-contenv bash
 
-exec s6-setuidgid app \
+exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
     /usr/bin/python3 /app/manage.py process_tasks

From b40626f9dca20b76b52cf94c536a03abb118c121 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 27 Jan 2025 20:24:23 -0500
Subject: [PATCH 130/143] Guess at ffmpeg options based on filename

---
 tubesync/sync/youtube.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 1550798c..c9cc632e 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -245,6 +245,22 @@ def download_media(url, media_format, extension, output_file, info_json,
         'home': os.path.dirname(output_file),
     })
 
+    codec_options = []
+    ofn = os.path.basename(output_file)
+    if 'av1-' in ofn:
+        codec_options = ['-c:v', 'libsvtav1', '-preset', '8', '-crf', '35']
+    elif 'vp9-' in ofn:
+        codec_options = ['-c:v', 'libvpx-vp9', '-b:v', '0', '-crf', '31']
+    ytopts['postprocessor_args'] = opts.get('postprocessor_args', {})
+    set_ffmpeg_codec = not (
+        ytopts['postprocessor_args'] and
+        ytopts['postprocessor_args']['modifychapters+ffmpeg']
+    )
+    if set_ffmpeg_codec and codec_options:
+        ytopts['postprocessor_args'].update({
+            'modifychapters+ffmpeg': codec_options,
+        })
+
     # clean-up incompatible keys
     pp_opts = {k: v for k, v in pp_opts.items() if not k.startswith('_')}
 

From 803319fe9fa76a08d88f4efc585d59d7f8b11a90 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 01:00:35 -0500
Subject: [PATCH 131/143] Let deepcopy do more work for us

---
 tubesync/sync/youtube.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index c9cc632e..5406f313 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -212,8 +212,8 @@ def download_media(url, media_format, extension, output_file, info_json,
     hook.download_progress = 0
 
     default_opts = yt_dlp.parse_options([]).options
-    pp_opts = deepcopy(default_opts.__dict__)
-    pp_opts.update({
+    pp_opts = deepcopy(default_opts)
+    pp_opts.__dict__.update({
         'embedthumbnail': embed_thumbnail,
         'addmetadata': embed_metadata,
         'addchapters': True,
@@ -222,8 +222,8 @@ def download_media(url, media_format, extension, output_file, info_json,
     })
 
     if skip_sponsors:
-        pp_opts['sponsorblock_mark'].update('all,-chapter'.split(','))
-        pp_opts['sponsorblock_remove'].update(sponsor_categories or {})
+        pp_opts.sponsorblock_mark.update('all,-chapter'.split(','))
+        pp_opts.sponsorblock_remove.update(sponsor_categories or {})
 
     ytopts = {
         'format': media_format,
@@ -261,13 +261,6 @@ def download_media(url, media_format, extension, output_file, info_json,
             'modifychapters+ffmpeg': codec_options,
         })
 
-    # clean-up incompatible keys
-    pp_opts = {k: v for k, v in pp_opts.items() if not k.startswith('_')}
-
-    # convert dict to namedtuple
-    yt_dlp_opts = namedtuple('yt_dlp_opts', pp_opts)
-    pp_opts = yt_dlp_opts(**pp_opts)
-
     # create the post processors list
     ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts))
 

From 06950366109ec2325ac26fb66bee823670b70925 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 01:45:45 -0500
Subject: [PATCH 132/143] Tweak the behavior of post processors

---
 tubesync/sync/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 5406f313..527da496 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -218,7 +218,9 @@ def download_media(url, media_format, extension, output_file, info_json,
         'addmetadata': embed_metadata,
         'addchapters': True,
         'embed_infojson': False,
+        'writethumbnail': False,
         'force_keyframes_at_cuts': True,
+        'sponskrub': False,
     })
 
     if skip_sponsors:

From bf7127b973f4f4c9a385fa55ac7243ef0d5a8220 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:19:17 -0500
Subject: [PATCH 133/143] Schedule the rename task when only the settings allow
 it

---
 tubesync/sync/signals.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 30f6694f..0b19c360 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -54,7 +54,7 @@ def source_post_save(sender, instance, created, **kwargs):
         if instance.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST and instance.copy_channel_images:
             download_source_images(
                 str(instance.pk),
-                priority=0,
+                priority=2,
                 verbose_name=verbose_name.format(instance.name)
             )
         if instance.index_schedule > 0:
@@ -69,17 +69,28 @@ def source_post_save(sender, instance, created, **kwargs):
                 verbose_name=verbose_name.format(instance.name),
                 remove_existing_tasks=True
             )
-    verbose_name = _('Renaming all media for source "{}"')
-    rename_all_media_for_source(
-        str(instance.pk),
-        priority=0,
-        verbose_name=verbose_name.format(instance.name),
-        remove_existing_tasks=True
+    # Check settings before any rename tasks are scheduled
+    rename_sources_setting = settings.RENAME_SOURCES or list()
+    create_rename_task = (
+        (
+            instance.directory and
+            instance.directory in rename_sources_setting
+        ) or
+        settings.RENAME_ALL_SOURCES
     )
+    if create_rename_task:
+        verbose_name = _('Renaming all media for source "{}"')
+        rename_all_media_for_source(
+            str(instance.pk),
+            queue=str(instance.pk),
+            priority=1,
+            verbose_name=verbose_name.format(instance.name),
+            remove_existing_tasks=False
+        )
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
-        priority=1,
+        priority=2,
         verbose_name=verbose_name.format(instance.name),
         remove_existing_tasks=True
     )

From d9c73d3dd73288f5846e8af084614e59cac9be6e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:28:21 -0500
Subject: [PATCH 134/143] Add new container settings

- TUBESYNC_RENAME_ALL_SOURCES: True or False
- TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories
---
 tubesync/tubesync/local_settings.py.container | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
index 0114e76d..1b974cdf 100644
--- a/tubesync/tubesync/local_settings.py.container
+++ b/tubesync/tubesync/local_settings.py.container
@@ -93,6 +93,14 @@ SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip(
 SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR )
 
 
+# TUBESYNC_RENAME_ALL_SOURCES: True or False
+RENAME_ALL_SOURCES_STR = os.getenv('TUBESYNC_RENAME_ALL_SOURCES', 'False').strip().lower()
+RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR )
+# TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories
+RENAME_SOURCES_STR = os.getenv('TUBESYNC_RENAME_SOURCES', '')
+RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None
+
+
 VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))
 
 

From e59ba88e4e3960d50412f4c4ef2f8a4ca0610eb5 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 16:29:45 -0500
Subject: [PATCH 135/143] Add default values for the new settings

---
 tubesync/tubesync/settings.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py
index a888e91d..3c350ab3 100644
--- a/tubesync/tubesync/settings.py
+++ b/tubesync/tubesync/settings.py
@@ -177,6 +177,10 @@ COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
 
 
+RENAME_ALL_SOURCES = False
+RENAME_SOURCES = None
+
+
 try:
     from .local_settings import *
 except ImportError as e:

From e3501fe77a13ae3ebd125cdc88402321255a5c52 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 19:12:56 -0500
Subject: [PATCH 136/143] Fix CommaSepChoiceField

Using the separator that was chosen is the biggest fix.
---
 tubesync/sync/fields.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py
index 2514c525..f44688b7 100644
--- a/tubesync/sync/fields.py
+++ b/tubesync/sync/fields.py
@@ -32,24 +32,28 @@ class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
 class CommaSepChoiceField(models.Field):
     "Implements comma-separated storage of lists"
 
+    # If 'text' isn't correct add the vendor override here.
+    _DB_TYPES = {}
+
     def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
-        self.separator = separator
+        super().__init__(*args, **kwargs)
+        self.separator = str(separator)
         self.possible_choices = possible_choices
         self.selected_choices = []
         self.allow_all = allow_all
         self.all_label = all_label
         self.all_choice = all_choice
-        super().__init__(*args, **kwargs)
 
     def deconstruct(self):
         name, path, args, kwargs = super().deconstruct()
-        if self.separator != ",":
+        if ',' != self.separator:
             kwargs['separator'] = self.separator
         kwargs['possible_choices'] = self.possible_choices
         return name, path, args, kwargs
 
     def db_type(self, connection):
-        return 'text'
+        value = self._DB_TYPES.get(connection.vendor, None)
+        return value if value is not None else 'text'
 
     def get_my_choices(self):
         choiceArray = []
@@ -72,21 +76,13 @@ class CommaSepChoiceField(models.Field):
                     'label': '',
                     'required': False}
         defaults.update(kwargs)
-        #del defaults.required
         return super().formfield(**defaults)
 
-    def deconstruct(self):
-        name, path, args, kwargs = super().deconstruct()
-        # Only include kwarg if it's not the default
-        if self.separator != ",":
-            kwargs['separator'] = self.separator
-        return name, path, args, kwargs
-
     def from_db_value(self, value, expr, conn):
-        if value is None:
+        if 0 == len(value) or value is None:
             self.selected_choices = []
         else:
-            self.selected_choices = value.split(",")
+            self.selected_choices = value.split(self.separator)
 
         return self
 
@@ -97,7 +93,7 @@ class CommaSepChoiceField(models.Field):
             return ""
 
         if self.all_choice not in value:
-            return ",".join(value)
+            return self.separator.join(value)
         else:
             return self.all_choice
 

From 2b5ad21b20cbae04ad389e1b49fd605254648e72 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 20:37:32 -0500
Subject: [PATCH 137/143] Use TextChoices for SponsorBlock_Category

---
 tubesync/sync/models.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index fd599d06..c83a5878 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -114,21 +114,26 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
-    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
-    SPONSORBLOCK_CATEGORIES_CHOICES = (
-        ('sponsor', 'Sponsor'),
-        ('intro', 'Intermission/Intro Animation'),
-        ('outro', 'Endcards/Credits'),
-        ('selfpromo', 'Unpaid/Self Promotion'),
-        ('preview', 'Preview/Recap'),
-        ('filler', 'Filler Tangent'),
-        ('interaction', 'Interaction Reminder'),
-        ('music_offtopic', 'Non-Music Section'),
-    )
+    # as stolen from:
+    # - https://wiki.sponsor.ajay.app/w/Types
+    # - https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+    #
+    # The spacing is a little odd, it is for easy copy/paste selection.
+    # Please don't change it.
+    # Every possible category fits in a string < 128 characters
+    class SponsorBlock_Category(models.TextChoices):
+        SPONSOR = 'sponsor', _( 'Sponsor' )
+        INTRO = 'intro', _( 'Intermission/Intro Animation' )
+        OUTRO = 'outro', _( 'Endcards/Credits' )
+        SELFPROMO = 'selfpromo', _( 'Unpaid/Self Promotion' )
+        PREVIEW = 'preview', _( 'Preview/Recap' )
+        FILLER = 'filler', _( 'Filler Tangent' )
+        INTERACTION = 'interaction', _( 'Interaction Reminder' )
+        MUSIC_OFFTOPIC = 'music_offtopic', _( 'Non-Music Section' )
 
     sponsorblock_categories = CommaSepChoiceField(
         _(''),
-        possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+        possible_choices=SponsorBlock_Category.choices,
         all_choice='all',
         allow_all=True,
         all_label='(all options)',

From f60c1d7b5b7b1aeec15a7c9863cc45c2a03a1244 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 28 Jan 2025 21:53:42 -0500
Subject: [PATCH 138/143] Move SponsorBlock_Category to fields.py

---
 tubesync/sync/fields.py | 19 +++++++++++++++++++
 tubesync/sync/models.py | 19 +------------------
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py
index 2514c525..c04cf5a2 100644
--- a/tubesync/sync/fields.py
+++ b/tubesync/sync/fields.py
@@ -3,6 +3,25 @@ from django.db import models
 from typing import Any, Optional, Dict
 from django.utils.translation import gettext_lazy as _
 
+
+# as stolen from:
+# - https://wiki.sponsor.ajay.app/w/Types
+# - https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+#
+# The spacing is a little odd, it is for easy copy/paste selection.
+# Please don't change it.
+# Every possible category fits in a string < 128 characters
+class SponsorBlock_Category(models.TextChoices):
+    SPONSOR = 'sponsor', _( 'Sponsor' )
+    INTRO = 'intro', _( 'Intermission/Intro Animation' )
+    OUTRO = 'outro', _( 'Endcards/Credits' )
+    SELFPROMO = 'selfpromo', _( 'Unpaid/Self Promotion' )
+    PREVIEW = 'preview', _( 'Preview/Recap' )
+    FILLER = 'filler', _( 'Filler Tangent' )
+    INTERACTION = 'interaction', _( 'Interaction Reminder' )
+    MUSIC_OFFTOPIC = 'music_offtopic', _( 'Non-Music Section' )
+
+
 # this is a form field!
 class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
     template_name = 'widgets/checkbox_select.html'
diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index c83a5878..13692a93 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -23,7 +23,7 @@ from .utils import seconds_to_timestr, parse_media_format, filter_response
 from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
-from .fields import CommaSepChoiceField
+from .fields import CommaSepChoiceField, SponsorBlock_Category
 
 media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 
@@ -114,23 +114,6 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
-    # as stolen from:
-    # - https://wiki.sponsor.ajay.app/w/Types
-    # - https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
-    #
-    # The spacing is a little odd, it is for easy copy/paste selection.
-    # Please don't change it.
-    # Every possible category fits in a string < 128 characters
-    class SponsorBlock_Category(models.TextChoices):
-        SPONSOR = 'sponsor', _( 'Sponsor' )
-        INTRO = 'intro', _( 'Intermission/Intro Animation' )
-        OUTRO = 'outro', _( 'Endcards/Credits' )
-        SELFPROMO = 'selfpromo', _( 'Unpaid/Self Promotion' )
-        PREVIEW = 'preview', _( 'Preview/Recap' )
-        FILLER = 'filler', _( 'Filler Tangent' )
-        INTERACTION = 'interaction', _( 'Interaction Reminder' )
-        MUSIC_OFFTOPIC = 'music_offtopic', _( 'Non-Music Section' )
-
     sponsorblock_categories = CommaSepChoiceField(
         _(''),
         possible_choices=SponsorBlock_Category.choices,

From 24ac2e1aed486890d9974810dee0c2e87a7ed096 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 29 Jan 2025 00:23:02 -0500
Subject: [PATCH 139/143] Field expects verbose_name to be first, not separator

---
 tubesync/sync/fields.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py
index f44688b7..3a9044f1 100644
--- a/tubesync/sync/fields.py
+++ b/tubesync/sync/fields.py
@@ -35,7 +35,7 @@ class CommaSepChoiceField(models.Field):
     # If 'text' isn't correct add the vendor override here.
     _DB_TYPES = {}
 
-    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
+    def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs):
         super().__init__(*args, **kwargs)
         self.separator = str(separator)
         self.possible_choices = possible_choices

From 90eca25c27c0094c0a8d67a1ee622ce35f1613b6 Mon Sep 17 00:00:00 2001
From: meeb <meeb@meeb.org>
Date: Wed, 29 Jan 2025 17:15:29 +1100
Subject: [PATCH 140/143] add migrations for sponsorblock categories reworking,
 part of #657

---
 ...27_alter_source_sponsorblock_categories.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py

diff --git a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py
new file mode 100644
index 00000000..92fbc98a
--- /dev/null
+++ b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2025-01-29 06:14
+
+from django.db import migrations
+import sync.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0026_alter_source_sub_langs'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''),
+        ),
+    ]

From 78b9aadcbfa4cf41772cfad041fb47c4eed3e55c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 29 Jan 2025 03:55:52 -0500
Subject: [PATCH 141/143] Use a temporary directory per task

I am seeing multiple ffmpeg processes that mess each other up.

I do not know why this is happening yet, but for now let them operate in their own temporary directories.
---
 tubesync/sync/youtube.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 4ca78ae9..32658ac4 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -8,8 +8,10 @@ import os
 
 from collections import namedtuple
 from common.logger import log
-from copy import copy, deepcopy
+from copy import deepcopy
 from pathlib import Path
+from tempfile import TemporaryDirectory
+from urllib.parse import urlsplit, parse_qs
 
 from django.conf import settings
 from .utils import mkdir_p
@@ -41,7 +43,7 @@ class YouTubeError(yt_dlp.utils.DownloadError):
 
 
 def get_yt_opts():
-    opts = copy(_defaults)
+    opts = deepcopy(_defaults)
     cookie_file = settings.COOKIES_FILE
     if cookie_file.is_file():
         cookie_file_path = str(cookie_file.resolve())
@@ -244,12 +246,22 @@ def download_media(url, media_format, extension, output_file, info_json,
     }
     opts = get_yt_opts()
     ytopts['paths'] = opts.get('paths', {})
+    output_dir = os.path.dirname(output_file)
+    temp_dir_parent = output_dir
+    temp_dir_prefix = '.yt_dlp-'
+    if 'temp' in ytopts['paths']:
+        v_key = parse_qs(urlsplit(url).query).get('v').pop()
+        temp_dir_parent = ytopts['paths']['temp']
+        temp_dir_prefix = f'{temp_dir_prefix}{v_key}-'
+    temp_dir = TemporaryDirectory(prefix=temp_dir_prefix,dir=temp_dir_parent)
+    (Path(temp_dir) / '.ignore').touch(exist_ok=True)
     ytopts['paths'].update({
-        'home': os.path.dirname(output_file),
+        'home': output_dir,
+        'temp': temp_dir,
     })
 
     codec_options = []
-    ofn = os.path.basename(output_file)
+    ofn = ytopts['outtmpl']
     if 'av1-' in ofn:
         codec_options = ['-c:v', 'libsvtav1', '-preset', '8', '-crf', '35']
     elif 'vp9-' in ofn:

From b816eb3596546eea672b6194e6a6958e049f280e Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 29 Jan 2025 04:05:18 -0500
Subject: [PATCH 142/143] fixup: use the name attribute

---
 tubesync/sync/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 32658ac4..c0360ca9 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -254,10 +254,10 @@ def download_media(url, media_format, extension, output_file, info_json,
         temp_dir_parent = ytopts['paths']['temp']
         temp_dir_prefix = f'{temp_dir_prefix}{v_key}-'
     temp_dir = TemporaryDirectory(prefix=temp_dir_prefix,dir=temp_dir_parent)
-    (Path(temp_dir) / '.ignore').touch(exist_ok=True)
+    (Path(temp_dir.name) / '.ignore').touch(exist_ok=True)
     ytopts['paths'].update({
         'home': output_dir,
-        'temp': temp_dir,
+        'temp': temp_dir.name,
     })
 
     codec_options = []

From 3822c1a4f29eefc26d9b3507b4dd7863bd3514c7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 29 Jan 2025 21:13:17 -0500
Subject: [PATCH 143/143] Add `ytopts`

These are all optimizations:
- `writethumbnail`: for EmbedThumbnail to use
- `check_formats`: to save requests
- `overwrites`: to keep the first version of the video
- sleep intervals to slow down our tasks only for video downloads
---
 tubesync/sync/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index c0360ca9..259beea3 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -243,6 +243,12 @@ def download_media(url, media_format, extension, output_file, info_json,
         'writesubtitles': write_subtitles,
         'writeautomaticsub': auto_subtitles,
         'subtitleslangs': sub_langs.split(','),
+        'writethumbnail': True,
+        'check_formats': False,
+        'overwrites': None,
+        'sleep_interval': 30,
+        'max_sleep_interval': 600,
+        'sleep_interval_requests': 30,
     }
     opts = get_yt_opts()
     ytopts['paths'] = opts.get('paths', {})