From e01d155b10fde284500804fca33fc83fcef38afe Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 15:55:24 -0400 Subject: [PATCH 01/11] Switch to `getattr` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 6ee64747..ac325073 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -221,7 +221,7 @@ def media_post_save(sender, instance, created, **kwargs): else: # Downloaded media might need to be renamed # Check settings before any rename tasks are scheduled - rename_sources_setting = settings.RENAME_SOURCES or list() + rename_sources_setting = getattr(settings, 'RENAME_SOURCES', list()) create_rename_task = ( ( media.source.directory and From afe0a75824a351cdf0159cc4ae6c0c788c80c32d Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 16:21:01 -0400 Subject: [PATCH 02/11] Handle `None` returned by `getattr` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index ac325073..4c332eca 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -221,7 +221,7 @@ def media_post_save(sender, instance, created, **kwargs): else: # Downloaded media might need to be renamed # Check settings before any rename tasks are scheduled - rename_sources_setting = getattr(settings, 'RENAME_SOURCES', list()) + rename_sources_setting = getattr(settings, 'RENAME_SOURCES') or list() create_rename_task = ( ( media.source.directory and From 033656b436db2ad3951e5944d17b5d9d68ead9b3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 16:23:48 -0400 Subject: [PATCH 03/11] Be consistent in the task also --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 34c37e6c..d29b080c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -733,7 +733,7 @@ def rename_all_media_for_source(source_id): f'source exists with ID: {source_id}') raise InvalidTaskError(_('no such source')) from e # Check that the settings allow renaming - rename_sources_setting = getattr(settings, 'RENAME_SOURCES', list()) + rename_sources_setting = getattr(settings, 'RENAME_SOURCES') or list() create_rename_tasks = ( ( source.directory and From 15b8d4b83fc8079630e57b55107a80f9fb711dbf Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 17:31:20 -0400 Subject: [PATCH 04/11] Add warning to the dashboard --- tubesync/sync/templates/sync/dashboard.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tubesync/sync/templates/sync/dashboard.html b/tubesync/sync/templates/sync/dashboard.html index af342800..23e1cdb2 100644 --- a/tubesync/sync/templates/sync/dashboard.html +++ b/tubesync/sync/templates/sync/dashboard.html @@ -99,6 +99,18 @@ +
+
+

Warnings

+
+ An upcoming release, after 2025-006-01, will introduce automated file renaming.
+ To prevent this change from taking effect, you can set an environment variable before that date.
+ See the GitHub README + for more details or ask questions using + issue #785.
+
+
+

Runtime information

From 55f55e73058f701f23c09da505792f1d634782a4 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 20:01:20 -0400 Subject: [PATCH 05/11] Extract audio from `webm` downloads --- tubesync/sync/youtube.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index a10ca31d..d990b0f0 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -14,6 +14,7 @@ from tempfile import TemporaryDirectory from urllib.parse import urlsplit, parse_qs from django.conf import settings +from .choices import Val, FileExtension from .hooks import postprocessor_hook, progress_hook from .utils import mkdir_p import yt_dlp @@ -301,6 +302,10 @@ def download_media( ).options.sponsorblock_mark pp_opts.sponsorblock_remove.update(sponsor_categories or {}) + if Val(FileExtension.OGG) == extension: + pp_opts.extractaudio = True + pp_opts.nopostoverwrites = False + ytopts = { 'format': media_format, 'merge_output_format': extension, From e394232b15ef85ba39bd1a47d9ff1b524783077a Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 20:10:55 -0400 Subject: [PATCH 06/11] Use a set of audio-only extensions --- tubesync/sync/youtube.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index d990b0f0..921c664b 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -302,7 +302,12 @@ def download_media( ).options.sponsorblock_mark pp_opts.sponsorblock_remove.update(sponsor_categories or {}) - if Val(FileExtension.OGG) == extension: + # Enable audio extraction for audio-only extensions + audio_exts = { + Val(FileExtension.M4A), + Val(FileExtension.OGG), + } + if extension in audio_exts: pp_opts.extractaudio = True pp_opts.nopostoverwrites = False From aa78c7309e69e1d28dc116956fcf2df8f32cda60 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 20:13:20 -0400 Subject: [PATCH 07/11] Use a single `Val` call --- tubesync/sync/youtube.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 921c664b..145e4c5d 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -303,10 +303,10 @@ def download_media( pp_opts.sponsorblock_remove.update(sponsor_categories or {}) # Enable audio extraction for audio-only extensions - audio_exts = { - Val(FileExtension.M4A), - Val(FileExtension.OGG), - } + audio_exts = set(Val( + FileExtension.M4A, + FileExtension.OGG, + )) if extension in audio_exts: pp_opts.extractaudio = True pp_opts.nopostoverwrites = False From 7a0fdd16cd6e935fd39295adc5b6b2949dd00e1b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 21:11:08 -0400 Subject: [PATCH 08/11] Remove unused `media_post_delete` --- tubesync/sync/management/commands/delete-source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 5ab8a325..2f149a67 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import signals from common.logger import log from sync.models import Source, Media, MediaServer -from sync.signals import media_post_delete from sync.tasks import schedule_media_servers_update From c145987b0f3e5009a541e90bba113d84327f6cca Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 00:04:08 -0400 Subject: [PATCH 09/11] Use distinct transactions --- .../sync/management/commands/reset-tasks.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index d7818007..3d6f515d 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -13,21 +13,28 @@ class Command(BaseCommand): help = 'Resets all tasks' - @atomic(durable=True) def handle(self, *args, **options): log.info('Resettings all tasks...') - # Delete all tasks - Task.objects.all().delete() - # Iter all tasks - for source in Source.objects.all(): - # Recreate the initial indexing task - log.info(f'Resetting tasks for source: {source}') - verbose_name = _('Index media from source "{}"') - index_source_task( - str(source.pk), - repeat=source.index_schedule, - verbose_name=verbose_name.format(source.name) - ) - # This also chains down to call each Media objects .save() as well - source.save() + with atomic(durable=True): + # Delete all tasks + Task.objects.all().delete() + # Iter all sources, creating new tasks + for source in Source.objects.all(): + verbose_name = _('Check download directory exists for source "{}"') + check_source_directory_exists( + str(source.pk), + verbose_name=verbose_name.format(source.name), + ) + # Recreate the initial indexing task + log.info(f'Resetting tasks for source: {source}') + verbose_name = _('Index media from source "{}"') + index_source_task( + str(source.pk), + repeat=source.index_schedule, + verbose_name=verbose_name.format(source.name), + ) + with atomic(durable=True): + for source in Source.objects.all(): + # This also chains down to call each Media objects .save() as well + source.save() log.info('Done') From 6278030a9bb0c7a89df2f60f1b9a5fdd7b6e5db9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 00:10:26 -0400 Subject: [PATCH 10/11] Check source directory when tasks were reset --- tubesync/sync/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index f489144b..0e3f8dbb 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -931,6 +931,11 @@ class ResetTasks(FormView): Task.objects.all().delete() # Iter all tasks for source in Source.objects.all(): + verbose_name = _('Check download directory exists for source "{}"') + check_source_directory_exists( + str(source.pk), + verbose_name=verbose_name.format(source.name), + ) # Recreate the initial indexing task verbose_name = _('Index media from source "{}"') index_source_task( From 60ce61bfd8202a0fef5fa8c40e11a10aa56fe23a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 00:41:57 -0400 Subject: [PATCH 11/11] Increase minor version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 0ac2b462..fdf42c3a 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.13.7' +VERSION = '0.14.1' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = []