From 85fb479c5ead27fcb256ea7a9e45658ee2bcf716 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 17:22:27 -0400 Subject: [PATCH 1/5] Better indexing of inactive sources --- tubesync/sync/tasks.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index dfec330d..4d9d4c7c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -190,6 +190,23 @@ def index_source_task(source_id): except Source.DoesNotExist: # Task triggered but the Source has been deleted, delete the task return + # An inactive Source would return an empty list for videos anyway + if not source.is_active: + cleanup_completed_tasks() + # deleting expired media should still happen when an index task is requested + with atomic(durable=True): + cleanup_old_media() + # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') + for mediaserver in MediaServer.objects.all(): + rescan_media_server( + str(mediaserver.pk), + priority=30, + verbose_name=verbose_name.format(mediaserver), + remove_existing_tasks=True, + ) + return # Reset any errors source.has_failed = False source.save() @@ -254,6 +271,17 @@ def index_source_task(source_id): log.info(f'Cleaning up media no longer in source: {source}') cleanup_removed_media(source, videos) + # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') + for mediaserver in MediaServer.objects.all(): + rescan_media_server( + str(mediaserver.pk), + priority=30, + verbose_name=verbose_name.format(mediaserver), + remove_existing_tasks=True, + ) + @background(schedule=0) def check_source_directory_exists(source_id): From 34eea62c847daa4c5710910090b99a9114368009 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 17:33:28 -0400 Subject: [PATCH 2/5] Don't log inside the loop --- tubesync/sync/signals.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 77e5686e..9284ea97 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -377,14 +377,13 @@ def media_post_delete(sender, instance, **kwargs): if not instance.source.is_active: return # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') for mediaserver in MediaServer.objects.all(): - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') rescan_media_server( str(mediaserver.pk), - schedule=5, - priority=0, + priority=30, verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True + remove_existing_tasks=True, ) From 7e721c98a7ffbed8d4be12f3103b72220b4ee31c Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 17:42:06 -0400 Subject: [PATCH 3/5] Don't update media servers for every Media item --- tubesync/sync/signals.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 9284ea97..be848a0a 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -374,16 +374,3 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) - if not instance.source.is_active: - return - # Schedule a task to update media servers - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all(): - rescan_media_server( - str(mediaserver.pk), - priority=30, - verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True, - ) - From 17b82d426472e7f621b5fff732a7ba333b94f2c1 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 18:13:24 -0400 Subject: [PATCH 4/5] Schedule update of media servers after deletion loops --- tubesync/sync/tasks.py | 88 ++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4d9d4c7c..d502f904 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -160,24 +160,47 @@ def cleanup_completed_tasks(): CompletedTask.objects.filter(run_at__lt=delta).delete() +def schedule_media_servers_update(): + with atomic(): + # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') + for mediaserver in MediaServer.objects.all(): + rescan_media_server( + str(mediaserver.pk), + priority=30, + verbose_name=verbose_name.format(mediaserver), + remove_existing_tasks=True, + ) + + def cleanup_old_media(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): - delta = timezone.now() - timedelta(days=source.days_to_keep) - for media in source.media_source.filter(downloaded=True, download_date__lt=delta): - log.info(f'Deleting expired media: {source} / {media} ' - f'(now older than {source.days_to_keep} days / ' - f'download_date before {delta})') - # .delete() also triggers a pre_delete signal that removes the files - media.delete() + with atomic(): + for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): + delta = timezone.now() - timedelta(days=source.days_to_keep) + for media in source.media_source.filter(downloaded=True, download_date__lt=delta): + log.info(f'Deleting expired media: {source} / {media} ' + f'(now older than {source.days_to_keep} days / ' + f'download_date before {delta})') + with atomic(): + # .delete() also triggers a pre_delete/post_delete signals that remove files + media.delete() + schedule_media_servers_update() def cleanup_removed_media(source, videos): - media_objects = Media.objects.filter(source=source) - for media in media_objects: - matching_source_item = [video['id'] for video in videos if video['id'] == media.key] - if not matching_source_item: - log.info(f'{media.name} is no longer in source, removing') - media.delete() + if not source.delete_removed_media: + return + log.info(f'Cleaning up media no longer in source: {source}') + with atomic(durable=True): + media_objects = Media.objects.filter(source=source) + for media in media_objects: + matching_source_item = [video['id'] for video in videos if video['id'] == media.key] + if not matching_source_item: + log.info(f'{media.name} is no longer in source, removing') + with atomic(): + media.delete() + schedule_media_servers_update() @background(schedule=300, remove_existing_tasks=True) @@ -185,6 +208,7 @@ def index_source_task(source_id): ''' Indexes media available from a Source object. ''' + cleanup_completed_tasks() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist: @@ -192,20 +216,8 @@ def index_source_task(source_id): return # An inactive Source would return an empty list for videos anyway if not source.is_active: - cleanup_completed_tasks() # deleting expired media should still happen when an index task is requested - with atomic(durable=True): - cleanup_old_media() - # Schedule a task to update media servers - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all(): - rescan_media_server( - str(mediaserver.pk), - priority=30, - verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True, - ) + cleanup_old_media() return # Reset any errors source.has_failed = False @@ -262,25 +274,9 @@ def index_source_task(source_id): priority=20, verbose_name=verbose_name.format(media.pk), ) - # Tack on a cleanup of old completed tasks - cleanup_completed_tasks() - with atomic(durable=True): - # Tack on a cleanup of old media - cleanup_old_media() - if source.delete_removed_media: - log.info(f'Cleaning up media no longer in source: {source}') - cleanup_removed_media(source, videos) - - # Schedule a task to update media servers - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all(): - rescan_media_server( - str(mediaserver.pk), - priority=30, - verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True, - ) + # Cleanup of old downloaded media and media no longer available from the source + cleanup_old_media() + cleanup_removed_media(source, videos) @background(schedule=0) From 8f9fbb9a4cb856c994f1ea63789ef9d2b366a82a Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 18:25:31 -0400 Subject: [PATCH 5/5] Call `cleanup_removed_media` from within the transaction --- tubesync/sync/tasks.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d502f904..b3850a32 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -192,14 +192,13 @@ def cleanup_removed_media(source, videos): if not source.delete_removed_media: return log.info(f'Cleaning up media no longer in source: {source}') - with atomic(durable=True): - media_objects = Media.objects.filter(source=source) - for media in media_objects: - matching_source_item = [video['id'] for video in videos if video['id'] == media.key] - if not matching_source_item: - log.info(f'{media.name} is no longer in source, removing') - with atomic(): - media.delete() + media_objects = Media.objects.filter(source=source) + for media in media_objects: + matching_source_item = [video['id'] for video in videos if video['id'] == media.key] + if not matching_source_item: + log.info(f'{media.name} is no longer in source, removing') + with atomic(): + media.delete() schedule_media_servers_update() @@ -209,6 +208,8 @@ def index_source_task(source_id): Indexes media available from a Source object. ''' cleanup_completed_tasks() + # deleting expired media should happen any time an index task is requested + cleanup_old_media() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist: @@ -216,8 +217,6 @@ def index_source_task(source_id): return # An inactive Source would return an empty list for videos anyway if not source.is_active: - # deleting expired media should still happen when an index task is requested - cleanup_old_media() return # Reset any errors source.has_failed = False @@ -274,9 +273,8 @@ def index_source_task(source_id): priority=20, verbose_name=verbose_name.format(media.pk), ) - # Cleanup of old downloaded media and media no longer available from the source - cleanup_old_media() - cleanup_removed_media(source, videos) + # Cleanup of media no longer available from the source + cleanup_removed_media(source, videos) @background(schedule=0)