diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index fdc954a3..4a1884d8 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -17,6 +17,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.utils import timezone +from django.db.transaction import atomic from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ from background_task import background @@ -179,6 +180,7 @@ def cleanup_removed_media(source, videos): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -221,7 +223,8 @@ def index_source_task(source_id): if published_dt is not None: media.published = published_dt try: - media.save() + with atomic(): + media.save() log.debug(f'Indexed media: {source} / {media}') # log the new media instances new_media_instance = ( @@ -611,9 +614,10 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - for media in mqs: - if media.uuid not in already_saved: - media.save() + with atomic(): + for media in mqs: + if media.uuid not in already_saved: + media.save() @background(schedule=60, remove_existing_tasks=True) @@ -626,6 +630,7 @@ def rename_media(media_id): @background(schedule=300, remove_existing_tasks=True) +@atomic(durable=True) def rename_all_media_for_source(source_id): try: source = Source.objects.get(pk=source_id) @@ -653,7 +658,8 @@ def rename_all_media_for_source(source_id): downloaded=True, ) for media in mqs: - media.rename_files() + with atomic(): + media.rename_files() @background(schedule=60, remove_existing_tasks=True) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 95eebb8a..48cff0c9 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -146,6 +146,14 @@ def get_media_info(url, days=None): f'yesterday-{days!s}days' if days else None ) opts = get_yt_opts() + paths = opts.get('paths', dict()) + if 'temp' in paths: + temp_dir_obj = TemporaryDirectory(prefix='.yt_dlp-', dir=paths['temp']) + temp_dir_path = Path(temp_dir_obj.name) + (temp_dir_path / '.ignore').touch(exist_ok=True) + paths.update({ + 'temp': str(temp_dir_path), + }) opts.update({ 'ignoreerrors': False, # explicitly set this to catch exceptions 'ignore_no_formats_error': False, # we must fail first to try again with this enabled @@ -156,10 +164,14 @@ def get_media_info(url, days=None): 'check_formats': True, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { - 'youtube': {'formats': ['missing_pot']}, 'youtubetab': {'approximate_date': ['true']}, }, + 'paths': paths, + 'sleep_interval_requests': 2, + 'verbose': True if settings.DEBUG else False, }) + if start: + log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}') response = {} with yt_dlp.YoutubeDL(opts) as y: try: