From 365ad555dbd53e2f3321db1b391fc453bef839d5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Feb 2025 06:21:39 -0500 Subject: [PATCH 1/6] Skip downloaded media that was deleted Moved the save of media to the end of the function. --- tubesync/sync/signals.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 4f811add..4a3a8517 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -151,11 +151,6 @@ def media_post_save(sender, instance, created, **kwargs): if instance.can_download: instance.can_download = False can_download_changed = True - # Save the instance if any changes were required - if skip_changed or can_download_changed: - post_save.disconnect(media_post_save, sender=Media) - instance.save() - post_save.connect(media_post_save, sender=Media) # If the media is missing metadata schedule it to be downloaded if not instance.metadata and not instance.skip and not get_media_metadata_task(instance.pk): log.info(f'Scheduling task to download metadata for: {instance.url}') @@ -185,6 +180,10 @@ def media_post_save(sender, instance, created, **kwargs): ) # If the media has not yet been downloaded schedule it to be downloaded if not instance.media_file_exists: + # The file was deleted after it was downloaded, skip this media. + if instance.can_download and instance.downloaded: + skip_changed = True != instance.skip + instance.skip = True instance.downloaded = False instance.media_file = None if (not instance.downloaded and instance.can_download and not instance.skip @@ -198,6 +197,11 @@ def media_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) + # Save the instance if any changes were required + if skip_changed or can_download_changed: + post_save.disconnect(media_post_save, sender=Media) + instance.save() + post_save.connect(media_post_save, sender=Media) @receiver(pre_delete, sender=Media) From 09fab821e9095ef9a939c0363b9c0074660f449b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Feb 2025 01:34:28 -0500 Subject: [PATCH 2/6] Deduplicated `fields` --- tubesync/sync/views.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 24ff6866..39a8ac33 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -332,12 +332,24 @@ class ValidateSourceView(FormView): class EditSourceMixin: model = Source + # ordered by the model + fields = [ + k for k in dict.fromkeys(map( + lambda f: str(f).rsplit('.', 1)[-1], + Source._meta.fields + )).keys() if k not in frozenset(( + 'has_failed', + 'created', + 'last_crawl', + 'uuid' + )) + ] + # manual ordering fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', 'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', - 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images', - 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', - 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images', + 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', + 'prefer_60fps', 'prefer_hdr', 'fallback', + 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') From 00e89623be4d3ed106e0b18f63b65d1f0f9a4d37 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Feb 2025 01:56:07 -0500 Subject: [PATCH 3/6] Sort the excluded set --- tubesync/sync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 39a8ac33..695819f7 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -338,8 +338,8 @@ class EditSourceMixin: lambda f: str(f).rsplit('.', 1)[-1], Source._meta.fields )).keys() if k not in frozenset(( - 'has_failed', 'created', + 'has_failed', 'last_crawl', 'uuid' )) From 57e52f4848563353b73676cd0df83065192d7858 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Feb 2025 14:56:27 -0500 Subject: [PATCH 4/6] Remove the generated `fields` variant --- tubesync/sync/views.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 695819f7..84514262 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -332,18 +332,6 @@ class ValidateSourceView(FormView): class EditSourceMixin: model = Source - # ordered by the model - fields = [ - k for k in dict.fromkeys(map( - lambda f: str(f).rsplit('.', 1)[-1], - Source._meta.fields - )).keys() if k not in frozenset(( - 'created', - 'has_failed', - 'last_crawl', - 'uuid' - )) - ] # manual ordering fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', 'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media', From c34700839b41b7b51854d3f6df0baa5e6fdb8c31 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 11 Feb 2025 20:53:20 -0500 Subject: [PATCH 5/6] We can filter before metadata After #704, the `duration`, `published`, and `title` are all available before `metadata` has been retrieved. --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index fe245be5..bfc73696 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -137,7 +137,7 @@ def media_post_save(sender, instance, created, **kwargs): can_download_changed = False # Reset the skip flag if the download cap has changed if the media has not # already been downloaded - if not instance.downloaded and instance.metadata: + if not instance.downloaded: skip_changed = filter_media(instance) # Recalculate the "can_download" flag, this may From 69095f9f924321b4e914f5637e0e257cf548e154 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 12 Feb 2025 00:43:44 -0500 Subject: [PATCH 6/6] Use a raw string SyntaxWarning: invalid escape sequence '\/' --- tubesync/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 1c7b42e5..95efd9f3 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -112,8 +112,8 @@ def append_uri_params(uri, params): def clean_filename(filename): if not isinstance(filename, str): raise ValueError(f'filename must be a str, got {type(filename)}') - to_scrub = '<>\/:*?"|%' - for char in to_scrub: + to_scrub = r'<>\/:*?"|%' + for char in list(to_scrub): filename = filename.replace(char, '') clean_filename = '' for c in filename: