From d991b9593d8acbcd60be1794485dc1cde1925aec Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 07:59:34 -0400 Subject: [PATCH 01/17] Add `Source.is_active` property --- tubesync/sync/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 9ab126db..0ae25bd5 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -333,6 +333,15 @@ class Source(models.Model): replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') return slugify(replaced)[:80] + @property + def is_active(self): + active = ( + self.download_media or + self.index_streams or + self.index_videos + ) + return self.source.index_schedule and active + @property def is_audio(self): return self.source_resolution == SourceResolution.AUDIO.value From 016cb498abfb178ab4c4de8d2be20f7766071336 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 07:59:34 -0400 Subject: [PATCH 02/17] Add `Source.deactivate()` function --- tubesync/sync/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 0ae25bd5..7606d664 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -333,6 +333,18 @@ class Source(models.Model): replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') return slugify(replaced)[:80] + def deactivate(self): + self.download_media = False + self.index_streams = False + self.index_videos = False + self.index_schedule = IndexSchedule.NEVER + self.save(update_fields={ + 'download_media', + 'index_streams', + 'index_videos', + 'index_schedule', + }) + @property def is_active(self): active = ( From 5a46dce2a13bb6ad1bd6bb6910a6b6c766170182 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 08:11:37 -0400 Subject: [PATCH 03/17] Do not update media servers about media from inactive sources --- tubesync/sync/signals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 8bea1ce2..3738e3a9 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -344,6 +344,8 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) + if not instance.source.is_active + return # Schedule a task to update media servers for mediaserver in MediaServer.objects.all(): log.info(f'Scheduling media server updates') From 9d03c0d5d295112d6cff311fe2d798206c145e1e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 08:14:29 -0400 Subject: [PATCH 04/17] Do not detach the signal --- tubesync/sync/management/commands/delete-source.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 104ec887..5cdace18 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -29,8 +29,8 @@ class Command(BaseCommand): except Source.DoesNotExist: raise CommandError(f'Source does not exist with ' f'UUID: {source_uuid}') - # Detach post-delete signal for Media so we don't spam media servers - signals.post_delete.disconnect(media_post_delete, sender=Media) + # Reconfigure the source to not update the disk or media servers + source.deactivate() # Delete the source, triggering pre-delete signals for each media item log.info(f'Found source with UUID "{source.uuid}" with name ' f'"{source.name}" and deleting it, this may take some time!') @@ -45,7 +45,5 @@ class Command(BaseCommand): verbose_name=verbose_name.format(mediaserver), remove_existing_tasks=True ) - # Re-attach signals - signals.post_delete.connect(media_post_delete, sender=Media) # All done log.info('Done') From 247b9f2a72d9789f8fdb5adc1687f35a4700c214 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 08:15:41 -0400 Subject: [PATCH 05/17] Update the media server quicker than the default --- tubesync/sync/management/commands/delete-source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 5cdace18..98ff59b5 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -42,6 +42,7 @@ class Command(BaseCommand): rescan_media_server( str(mediaserver.pk), priority=0, + schedule=30, verbose_name=verbose_name.format(mediaserver), remove_existing_tasks=True ) From bdf9a69f48bc137fba27256503ab838e7eae29bb Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 08:24:41 -0400 Subject: [PATCH 06/17] Clean up my `git status` --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 17e61eba..c5cd63bc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ # C extensions *.so +# vim swap files +.*.swp + # Distribution / packaging .Python build/ From 453b9eaa79f205e99386d7e8b2de659450272e8c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 11:32:34 -0400 Subject: [PATCH 07/17] Mark the directory for removal after the Source is deleted --- tubesync/sync/signals.py | 6 ++++++ tubesync/sync/views.py | 12 ++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 3738e3a9..f4f0d25f 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -1,4 +1,5 @@ from pathlib import Path +from shutil import rmtree from tempfile import TemporaryDirectory from django.conf import settings from django.db.models.signals import pre_save, post_save, pre_delete, post_delete @@ -151,6 +152,11 @@ def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted log.info(f'Deleting tasks for source: {instance.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) + source = instance + # Remove the directory, if the user requested that + directory_path = Path(source.directory_path) + if (directory_path / '.to_be_removed').is_file(): + rmtree(directory_path, True) @receiver(task_failed, sender=Task) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 99844a39..4c8e672b 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -3,7 +3,6 @@ import os import json from base64 import b64decode import pathlib -import shutil import sys from django.conf import settings from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect @@ -415,15 +414,8 @@ class DeleteSourceView(DeleteView, FormMixin): delete_media = True if delete_media_val is not False else False if delete_media: source = self.get_object() - for media in Media.objects.filter(source=source): - if media.media_file: - file_path = media.media_file.path - matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*') - for file in matching_files: - delete_file(file) - directory_path = source.directory_path - if os.path.exists(directory_path): - shutil.rmtree(directory_path, True) + directory_path = pathlib.Path(source.directory_path) + (directory_path / '.to_be_removed').touch(exist_ok=True) return super().post(request, *args, **kwargs) def get_success_url(self): From 9f171025a635cbb908387289fec3bf1ea8726da8 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 11:33:32 -0400 Subject: [PATCH 08/17] Remove the index task before trying to delete the Source --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index f4f0d25f..66a30232 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -142,6 +142,8 @@ def source_post_save(sender, instance, created, **kwargs): def source_pre_delete(sender, instance, **kwargs): # Triggered before a source is deleted, delete all media objects to trigger # the Media models post_delete signal + log.info(f'Deleting tasks for source: {instance.name}') + delete_task_by_source('sync.tasks.index_source_task', instance.pk) for media in Media.objects.filter(source=instance): log.info(f'Deleting media for source: {instance.name} item: {media.name}') media.delete() @@ -150,8 +152,6 @@ def source_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Source) def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted - log.info(f'Deleting tasks for source: {instance.name}') - delete_task_by_source('sync.tasks.index_source_task', instance.pk) source = instance # Remove the directory, if the user requested that directory_path = Path(source.directory_path) From cbed39b798c9ebb80ad0a2c8d21b9ab00ec68d9d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 12:02:14 -0400 Subject: [PATCH 09/17] Log the Source directory path before deleting --- tubesync/sync/management/commands/delete-source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 98ff59b5..206aee7f 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -34,6 +34,7 @@ class Command(BaseCommand): # Delete the source, triggering pre-delete signals for each media item log.info(f'Found source with UUID "{source.uuid}" with name ' f'"{source.name}" and deleting it, this may take some time!') + log.info(f'Source directory: {source.directory_path}') source.delete() # Update any media servers for mediaserver in MediaServer.objects.all(): From 640f51fe9567d6911cea65553f36faa2be751553 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 13:00:08 -0400 Subject: [PATCH 10/17] Deactivate the Source before deletion --- tubesync/sync/signals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 66a30232..5f97c563 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -142,6 +142,8 @@ def source_post_save(sender, instance, created, **kwargs): def source_pre_delete(sender, instance, **kwargs): # Triggered before a source is deleted, delete all media objects to trigger # the Media models post_delete signal + log.info(f'Deactivating source: {instance.name}') + instance.deactivate() log.info(f'Deleting tasks for source: {instance.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) for media in Media.objects.filter(source=instance): From 81ab9e3c92b0034ec157778f9f56fb43cf3775d2 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 13:32:49 -0400 Subject: [PATCH 11/17] Add and use `delete_all_media_for_source` task --- tubesync/sync/signals.py | 19 ++++++++++++++++--- tubesync/sync/tasks.py | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 5f97c563..063aa545 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -146,9 +146,22 @@ def source_pre_delete(sender, instance, **kwargs): instance.deactivate() log.info(f'Deleting tasks for source: {instance.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) - for media in Media.objects.filter(source=instance): - log.info(f'Deleting media for source: {instance.name} item: {media.name}') - media.delete() + # Schedule deletion of media + verbose_name = _('Deleting all media for source "{}"') + delete_all_media_for_source( + str(instance.pk), + str(instance.name), + priority=1, + verbose_name=verbose_name.format(instance.name), + ) + # Try to do it all immediately + # If this is killed, the scheduled task should do the work instead. + delete_all_media_for_source.now( + str(instance.pk), + str(instance.name), + priority=0, + verbose_name=verbose_name.format(instance.name), + ) @receiver(post_delete, sender=Source) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 498d73fe..dbbd804a 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -690,3 +690,24 @@ def wait_for_media_premiere(media_id): media.title = _(f'Premieres in {hours(media.published - now)} hours') media.save() +@background(schedule=300, remove_existing_tasks=False) +@atomic(durable=True) +def delete_all_media_for_source(source_id, source_name): + source = None + try: + source = Source.objects.get(pk=source_id) + except Source.DoesNotExist: + # Task triggered but the source no longer exists, do nothing + log.error(f'Task delete_all_media_for_source(pk={source_id}) called but no ' + f'source exists with ID: {source_id}') + pass + mqs = Media.objects.all().defer( + 'metadata', + ).filter( + source=source or source_id, + ) + for media in mqs: + log.info(f'Deleting media for source: {source_name} item: {media.name}') + with atomic(): + media.delete() + From fc058a82f7b732c8b0b6f5a085809feafde74991 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 13:52:04 -0400 Subject: [PATCH 12/17] Map the new task --- tubesync/sync/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index dbbd804a..782db84c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -55,6 +55,7 @@ def map_task_to_instance(task): 'sync.tasks.rename_media': Media, 'sync.tasks.rename_all_media_for_source': Source, 'sync.tasks.wait_for_media_premiere': Media, + 'sync.tasks.delete_all_media_for_source': Source, } MODEL_URL_MAP = { Source: 'sync:source', From 054943854326ffe612eef4beaa6669109c534f8b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 15:16:31 -0400 Subject: [PATCH 13/17] fixup! Do not update media servers about media from inactive sources --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 063aa545..f1577f72 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -365,7 +365,7 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) - if not instance.source.is_active + if not instance.source.is_active: return # Schedule a task to update media servers for mediaserver in MediaServer.objects.all(): From 1d3e880c95802318437996272ed668c5397c8af6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 15:24:48 -0400 Subject: [PATCH 14/17] fixup! Add `Source.is_active` property --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 7606d664..176b69ee 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -352,7 +352,7 @@ class Source(models.Model): self.index_streams or self.index_videos ) - return self.source.index_schedule and active + return self.index_schedule and active @property def is_audio(self): From b18760082f58bb468a3070b5781acc49134a8e62 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 15:32:52 -0400 Subject: [PATCH 15/17] fixup! Add and use `delete_all_media_for_source` task --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index f1577f72..f6ad7826 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -13,8 +13,8 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, download_media, rescan_media_server, download_source_images, - save_all_media_for_source, rename_media, - get_media_metadata_task, get_media_download_task) + delete_all_media_for_source, save_all_media_for_source, + rename_media, get_media_metadata_task, get_media_download_task) from .utils import delete_file, glob_quote, mkdir_p from .filtering import filter_media from .choices import Val, YouTube_SourceType From 51a6a08f9c8768004f4b40ec369683784bbde23f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 15:42:03 -0400 Subject: [PATCH 16/17] This task is being called from a transaction already --- tubesync/sync/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 782db84c..d004362a 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -692,7 +692,6 @@ def wait_for_media_premiere(media_id): media.save() @background(schedule=300, remove_existing_tasks=False) -@atomic(durable=True) def delete_all_media_for_source(source_id, source_name): source = None try: From 7997b42ca656dc136c11844894c33573c813e2cf Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 14 Mar 2025 15:45:22 -0400 Subject: [PATCH 17/17] Do not use keyword arguments with the underlying function --- tubesync/sync/signals.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index f6ad7826..812d4c85 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -159,8 +159,6 @@ def source_pre_delete(sender, instance, **kwargs): delete_all_media_for_source.now( str(instance.pk), str(instance.name), - priority=0, - verbose_name=verbose_name.format(instance.name), )