From 9aa5e6b9675f4b0d9c501842baa010237927814b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:54:43 -0400 Subject: [PATCH 01/14] Create a media entry after deletion --- tubesync/sync/signals.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 32b0b5f6..8218c438 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,6 +295,25 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=instance.get_metadata_first_value( + 'thumbnail', + arg_dict=existing_metadata, + ), + ), + ) + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -370,3 +389,32 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) + # Create a media entry for the indexing task to find + # Requirements: + # source, key, duration, title, published + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) + if created: + old_metadata = instance.loaded_metadata + skipped_media.downloaded = False + skipped_media.duration = instance.duration + skipped_media.metadata = skipped_media.metadata_dumps( + arg_dict=dict( + _media_instance_was_deleted=True, + site=old_metadata.get('site'), + thumbnail=old_metadata.get('thumbnail'), + ), + ) + skipped_media.published = instance.published + skipped_media.title = instance.title + skipped_media.skip = True + skipped_media.manual_skip = True + skipped_media.save() + Metadata.objects.filter( + media__isnull=True, + site=old_metadata.get('site') or 'Youtube', + key=skipped_media.key, + ).update(media=skipped_media) + From f86b666c125446a116c1fddae5ecc90463bc3f52 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:57:23 -0400 Subject: [PATCH 02/14] fixup: import `Metadata` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 8218c438..49b0145d 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from .models import Source, Media, MediaServer +from .models import Source, Media, MediaServer, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, From d1fd568066a27c5804a097b34d47e57de3a141d6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:11:18 -0400 Subject: [PATCH 03/14] Test for the specific media items, not any task --- tubesync/sync/tests.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 303aa18a..b5440291 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,8 +469,19 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} - download_media_thumbnail_tasks = Task.objects.filter(**q) - self.assertFalse(download_media_thumbnail_tasks) + found_thumbnail_task1 = False + found_thumbnail_task2 = False + found_thumbnail_task3 = False + for task in Task.objects.filter(**q): + if test_media1_pk in task.task_params: + found_thumbnail_task1 = True + if test_media2_pk in task.task_params: + found_thumbnail_task2 = True + if test_media3_pk in task.task_params: + found_thumbnail_task3 = True + self.assertFalse(found_thumbnail_task1) + self.assertFalse(found_thumbnail_task2) + self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 4a04326d16737f9bbc7511ddc3c6c0e8f2129c92 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:31:49 -0400 Subject: [PATCH 04/14] Change the expected count and verify the key matches --- tubesync/sync/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b5440291..b84ffe03 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1847,5 +1847,6 @@ class TasksTestCase(TestCase): cleanup_old_media() self.assertEqual(src1.media_source.all().count(), 3) - self.assertEqual(src2.media_source.all().count(), 2) + self.assertEqual(src2.media_source.all().count(), 3) self.assertEqual(Media.objects.filter(pk=m22.pk).exists(), False) + self.assertEqual(Media.objects.filter(source=src2, key=m22.key, skip=True).exists(), True) From ba451d9401a0834a80b6d92f578e88497b704352 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:34:53 -0400 Subject: [PATCH 05/14] testing: sanity check --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 49b0145d..877cc4ed 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -396,7 +396,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if created: + if False and created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From a80623bce40e51a1755e4af448fb944d331cf964 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:41:14 -0400 Subject: [PATCH 06/14] testing: no save in pre-delete --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 877cc4ed..efb273a0 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,7 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - instance.save() + # TODO: instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) From 5a9c3050c8069d07092964af27a7682970b5dcfd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:50:37 -0400 Subject: [PATCH 07/14] Fixes from CI tests --- tubesync/sync/signals.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index efb273a0..26b18ecc 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,9 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - # TODO: instance.save() + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -396,7 +398,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if False and created: + if created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From 9d8a5574e416b0a9881e4777c44d8325e46ccb29 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:01:15 -0400 Subject: [PATCH 08/14] Delete tasks first then modify metadata --- tubesync/sync/signals.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 26b18ecc..76731598 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,27 +295,6 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): - # Save the metadata site & thumbnail URL to the metadata column - existing_metadata = instance.loaded_metadata - metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( - deleted=True, - site=instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail=instance.get_metadata_first_value( - 'thumbnail', - arg_dict=existing_metadata, - ), - ), - ) - # Do not create more tasks before deleting - instance.manual_skip = True - instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -331,6 +310,24 @@ def media_pre_delete(sender, instance, **kwargs): # Remove thumbnail file for deleted media if instance.thumb: instance.thumb.delete(save=False) + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=thumbnail_url, + ), + ) + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() @receiver(post_delete, sender=Media) From f7ca8189a0e61cf8e3f6b1f6c065c2f45d7fa5f1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:04:27 -0400 Subject: [PATCH 09/14] Update tests.py --- tubesync/sync/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b84ffe03..2926a296 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,10 +469,12 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} + download_media_thumbnail_tasks = Task.objects.filter(**q) + self.assertFalse(download_media_thumbnail_tasks) found_thumbnail_task1 = False found_thumbnail_task2 = False found_thumbnail_task3 = False - for task in Task.objects.filter(**q): + for task in download_media_thumbnail_tasks: if test_media1_pk in task.task_params: found_thumbnail_task1 = True if test_media2_pk in task.task_params: From 147f245e97d1cabd31955496679fedc87324ad1f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:06:28 -0400 Subject: [PATCH 10/14] Update tests.py --- tubesync/sync/tests.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 2926a296..24f0d092 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -471,19 +471,6 @@ class FrontEndTestCase(TestCase): q = {'task_name': 'sync.tasks.download_media_thumbnail'} download_media_thumbnail_tasks = Task.objects.filter(**q) self.assertFalse(download_media_thumbnail_tasks) - found_thumbnail_task1 = False - found_thumbnail_task2 = False - found_thumbnail_task3 = False - for task in download_media_thumbnail_tasks: - if test_media1_pk in task.task_params: - found_thumbnail_task1 = True - if test_media2_pk in task.task_params: - found_thumbnail_task2 = True - if test_media3_pk in task.task_params: - found_thumbnail_task3 = True - self.assertFalse(found_thumbnail_task1) - self.assertFalse(found_thumbnail_task2) - self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 7a8421e11acf0921eee516d13e7395934b6b1f01 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:10:14 -0400 Subject: [PATCH 11/14] Use the field mapping for site & thumbnail --- tubesync/sync/signals.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 76731598..9bec1401 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -314,16 +314,19 @@ def media_pre_delete(sender, instance, **kwargs): existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' column_metadata = instance.metadata_loads(metadata_str) + site_field = instance.get_metadata_field('extractor_key') + thumbnail_field = instance.get_metadata_field('thumbnail') instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( + arg_dict=dict(column_metadata).update(dict( deleted=True, - site=instance.get_metadata_first_value( + ).update({ + site_field: instance.get_metadata_first_value( 'extractor_key', 'Youtube', arg_dict=existing_metadata, ), - thumbnail=thumbnail_url, - ), + thumbnail_field: thumbnail_url, + })), ) # Do not create more tasks before deleting instance.manual_skip = True @@ -396,15 +399,19 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + site_field = instance.get_metadata_field('extractor_key') + thumbnail_url = instance.thumbnail + thumbnail_field = instance.get_metadata_field('thumbnail') old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration skipped_media.metadata = skipped_media.metadata_dumps( arg_dict=dict( _media_instance_was_deleted=True, - site=old_metadata.get('site'), - thumbnail=old_metadata.get('thumbnail'), - ), + ).update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }), ) skipped_media.published = instance.published skipped_media.title = instance.title @@ -413,7 +420,7 @@ def media_post_delete(sender, instance, **kwargs): skipped_media.save() Metadata.objects.filter( media__isnull=True, - site=old_metadata.get('site') or 'Youtube', + site=old_metadata.get(site_field) or 'Youtube', key=skipped_media.key, ).update(media=skipped_media) From 691d049e2cd74f36882c4bf9ae759f9f0d367e3b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:39:57 -0400 Subject: [PATCH 12/14] fixup: update `dict` first --- tubesync/sync/signals.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 9bec1401..1743914a 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,21 +313,18 @@ def media_pre_delete(sender, instance, **kwargs): # Save the metadata site & thumbnail URL to the metadata column existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) + arg_dict = instance.metadata_loads(metadata_str) site_field = instance.get_metadata_field('extractor_key') thumbnail_field = instance.get_metadata_field('thumbnail') - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update(dict( - deleted=True, - ).update({ - site_field: instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail_field: thumbnail_url, - })), - ) + arg_dict.update({ + site_field: instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail_field: thumbnail_url, + }) + instance.metadata = instance.metadata_dumps(arg_dict=arg_dict) # Do not create more tasks before deleting instance.manual_skip = True instance.save() @@ -399,19 +396,21 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key') thumbnail_url = instance.thumbnail thumbnail_field = instance.get_metadata_field('thumbnail') - old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration + arg_dict=dict( + _media_instance_was_deleted=True, + ) + arg_dict.update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }) skipped_media.metadata = skipped_media.metadata_dumps( - arg_dict=dict( - _media_instance_was_deleted=True, - ).update({ - site_field: old_metadata.get(site_field), - thumbnail_field: thumbnail_url, - }), + arg_dict=arg_dict, ) skipped_media.published = instance.published skipped_media.title = instance.title From 7beda36d87523c802c1b076925c85f6813e1d5a2 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:02:03 -0400 Subject: [PATCH 13/14] Mark media for deletion --- tubesync/sync/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2e6b1433..d29e8239 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -933,6 +933,10 @@ def delete_all_media_for_source(source_id, source_name, source_directory): for media in qs_gen(mqs): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): + #media.downloaded = False + media.skip = True + media.manual_skip = True + media.save() media.delete() # Remove the directory, if the user requested that directory_path = Path(source_directory) From 071508e2df128cae07ff3d0e922530627da7a4cd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:20:57 -0400 Subject: [PATCH 14/14] Coordination with deletion of `Source` instances --- tubesync/sync/signals.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 1743914a..6a2dd22f 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -333,7 +333,13 @@ def media_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): # Remove the video file, when configured to do so - if instance.source.delete_files_on_disk and instance.media_file: + remove_files = ( + instance.source and + instance.source.delete_files_on_disk and + instance.downloaded and + instance.media_file + ) + if remove_files: video_path = Path(str(instance.media_file.path)).resolve(strict=False) instance.media_file.delete(save=False) # the other files we created have these known suffixes @@ -391,10 +397,19 @@ def media_post_delete(sender, instance, **kwargs): # Create a media entry for the indexing task to find # Requirements: # source, key, duration, title, published - skipped_media, created = Media.objects.get_or_create( - key=instance.key, - source=instance.source, + created = False + create_for_indexing_task = ( + not ( + #not instance.downloaded and + instance.skip and + instance.manual_skip + ) ) + if create_for_indexing_task: + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) if created: old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key')