diff --git a/README.md b/README.md index 3eefaef8..e7568535 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ and less common features: > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. > > To prevent your installation from scheduling media file renaming tasks, -> you must set `TUBESYNC_RENAME_ALL_SOURCES=False` in the environment variables. +> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](../1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183). ### 2. Index frequency diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index b42542fe..2b055ce5 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -7,6 +7,8 @@ import os import json import math +import random +import time import uuid from io import BytesIO from hashlib import sha1 @@ -14,10 +16,11 @@ from pathlib import Path from datetime import datetime, timedelta from shutil import copyfile, rmtree from PIL import Image +from django import db from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import reset_queries, DatabaseError, IntegrityError +from django.db import DatabaseError, IntegrityError from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -35,6 +38,8 @@ from .utils import ( get_remote_image, resize_image_to_height, delete_file, write_text_file, filter_response, ) from .youtube import YouTubeError +db_vendor = db.connection.vendor + def get_hash(task_name, pk): ''' @@ -202,6 +207,20 @@ def migrate_queues(): return qs.update(queue=Val(TaskQueue.NET)) +def save_model(instance): + if 'sqlite' != db_vendor: + with atomic(durable=False): + instance.save() + return + + # work around for SQLite and its many + # "database is locked" errors + with atomic(durable=False): + instance.save() + arg = getattr(settings, 'SQLITE_DELAY_FLOAT', 1.5) + time.sleep(random.expovariate(arg)) + + @atomic(durable=False) def schedule_media_servers_update(): # Schedule a task to update media servers @@ -257,7 +276,7 @@ def index_source_task(source_id): ''' Indexes media available from a Source object. ''' - reset_queries() + db.reset_queries() cleanup_completed_tasks() # deleting expired media should happen any time an index task is requested cleanup_old_media() @@ -272,7 +291,7 @@ def index_source_task(source_id): # Reset any errors # TODO: determine if this affects anything source.has_failed = False - source.save() + save_model(source) # Index the source videos = source.index_media() if not videos: @@ -283,7 +302,7 @@ def index_source_task(source_id): f'is reachable') # Got some media, update the last crawl timestamp source.last_crawl = timezone.now() - source.save() + save_model(source) num_videos = len(videos) log.info(f'Found {num_videos} media items for source: {source}') fields = lambda f, m: m.get_metadata_field(f) @@ -490,7 +509,7 @@ def download_media_metadata(media_id): media.duration = media.metadata_duration # Don't filter media here, the post_save signal will handle that - media.save() + save_model(media) log.info(f'Saved {len(media.metadata)} bytes of metadata for: ' f'{source} / {media}: {media_id}') @@ -648,7 +667,7 @@ def download_media(media_id): media.downloaded_hdr = cformat['is_hdr'] else: media.downloaded_format = 'audio' - media.save() + save_model(media) # If selected, copy the thumbnail over as well if media.source.copy_thumbnails: if not media.thumb_file_exists: @@ -697,7 +716,7 @@ def save_all_media_for_source(source_id): source has its parameters changed and all media needs to be checked to see if its download status has changed. ''' - reset_queries() + db.reset_queries() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: @@ -742,14 +761,21 @@ def save_all_media_for_source(source_id): ) saved_later.add(media.uuid) + # Keep out of the way of the index task! + # SQLite will be locked for a while if we start + # a large source, which reschedules a more costly task. + if 'sqlite' == db_vendor: + index_task = get_source_index_task(source_id) + if index_task and index_task.locked_by_pid_running(): + raise Exception(_('Indexing not completed')) + # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{save_qs.count():,}' for mn, media in enumerate(qs_gen(save_qs), start=1): if media.uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) - with atomic(): - media.save() + save_model(media) # Reset task.verbose_name to the saved value update_task_status(task, None) @@ -766,8 +792,7 @@ def refresh_formats(media_id): log.debug(f'Failed to refresh formats for: {media.source} / {media.key}: {e!s}') pass else: - with atomic(): - media.save() + save_model(media) @background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.FS), remove_existing_tasks=True) @@ -826,15 +851,14 @@ def wait_for_media_premiere(media_id): if media.published < now: media.manual_skip = False media.skip = False - # start the download tasks after save + # the download tasks start after the media is saved else: media.manual_skip = True media.title = _(f'Premieres in {hours(media.published - now)} hours') task = get_media_premiere_task(media_id) if task: update_task_status(task, f'available in {hours(media.published - now)} hours') - with atomic(): - media.save() + save_model(media) @background(schedule=dict(priority=1, run_at=90), queue=Val(TaskQueue.FS), remove_existing_tasks=False) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 4f386b66..8b61692b 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -59,6 +59,12 @@ else: } DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"' + # the argument to random.expovariate(), + # a larger value means less delay + # with too little delay, you may see + # more "database is locked" errors + SQLITE_DELAY_FLOAT = 5 + DEFAULT_THREADS = 1 BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True)