From 58472f77858417dd2f408d9d2f8f2694fae83b3f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 21:47:48 -0400 Subject: [PATCH 1/4] Add explicit transactions for certain tasks --- tubesync/sync/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index fdc954a3..d85c6be7 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -17,6 +17,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.utils import timezone +from django.db.tansaction import atomic from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ from background_task import background @@ -179,6 +180,7 @@ def cleanup_removed_media(source, videos): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -221,7 +223,8 @@ def index_source_task(source_id): if published_dt is not None: media.published = published_dt try: - media.save() + with atomic(): + media.save() log.debug(f'Indexed media: {source} / {media}') # log the new media instances new_media_instance = ( @@ -611,9 +614,10 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - for media in mqs: - if media.uuid not in already_saved: - media.save() + with atomic(): + for media in mqs: + if media.uuid not in already_saved: + media.save() @background(schedule=60, remove_existing_tasks=True) @@ -626,6 +630,7 @@ def rename_media(media_id): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def rename_all_media_for_source(source_id): try: source = Source.objects.get(pk=source_id) @@ -653,7 +658,8 @@ def rename_all_media_for_source(source_id): downloaded=True, ) for media in mqs: - media.rename_files() + with atomic(): + media.rename_files() @background(schedule=60, remove_existing_tasks=True) From 84d42fb2ab1f94a17f99f4ce19b71e3251120515 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:02:44 -0400 Subject: [PATCH 2/4] Add more logging to `get_media_info` --- tubesync/sync/youtube.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 95eebb8a..a739a4f7 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -156,10 +156,13 @@ def get_media_info(url, days=None): 'check_formats': True, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { - 'youtube': {'formats': ['missing_pot']}, 'youtubetab': {'approximate_date': ['true']}, }, + 'sleep_interval_requests': 2, + 'verbose': True if settings.DEBUG else False, }) + if start: + log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}') response = {} with yt_dlp.YoutubeDL(opts) as y: try: From 4b3605f65ee46c4647a66ce7acab19258a9b6373 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:31:56 -0400 Subject: [PATCH 3/4] Use a temporary directory for testing formats --- tubesync/sync/youtube.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index a739a4f7..48cff0c9 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -146,6 +146,14 @@ def get_media_info(url, days=None): f'yesterday-{days!s}days' if days else None ) opts = get_yt_opts() + paths = opts.get('paths', dict()) + if 'temp' in paths: + temp_dir_obj = TemporaryDirectory(prefix='.yt_dlp-', dir=paths['temp']) + temp_dir_path = Path(temp_dir_obj.name) + (temp_dir_path / '.ignore').touch(exist_ok=True) + paths.update({ + 'temp': str(temp_dir_path), + }) opts.update({ 'ignoreerrors': False, # explicitly set this to catch exceptions 'ignore_no_formats_error': False, # we must fail first to try again with this enabled @@ -158,6 +166,7 @@ def get_media_info(url, days=None): 'extractor_args': { 'youtubetab': {'approximate_date': ['true']}, }, + 'paths': paths, 'sleep_interval_requests': 2, 'verbose': True if settings.DEBUG else False, }) From 79ed138aa103de6969b4220df3b6f36c1db97adf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 9 Mar 2025 22:45:49 -0400 Subject: [PATCH 4/4] fixup: typo --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d85c6be7..4a1884d8 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -17,7 +17,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.utils import timezone -from django.db.tansaction import atomic +from django.db.transaction import atomic from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ from background_task import background