Merge branch 'meeb:main' into patch-6

README.md

@@ -263,7 +263,7 @@ and less common features:
 > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`.
 > 
 > To prevent your installation from scheduling media file renaming tasks,
-> you must set `TUBESYNC_RENAME_ALL_SOURCES=False` in the environment variables.
+> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](../1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183).
 
 ### 2. Index frequency
 

tubesync/sync/tasks.py

@@ -7,6 +7,8 @@
 import os
 import json
 import math
+import random
+import time
 import uuid
 from io import BytesIO
 from hashlib import sha1
@@ -14,10 +16,11 @@ from pathlib import Path
 from datetime import datetime, timedelta
 from shutil import copyfile, rmtree
 from PIL import Image
+from django import db
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import SimpleUploadedFile
-from django.db import reset_queries, DatabaseError, IntegrityError
+from django.db import DatabaseError, IntegrityError
 from django.db.transaction import atomic
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -35,6 +38,8 @@ from .utils import ( get_remote_image, resize_image_to_height, delete_file,
                     write_text_file, filter_response, )
 from .youtube import YouTubeError
 
+db_vendor = db.connection.vendor
+
 
 def get_hash(task_name, pk):
     '''
@@ -202,6 +207,20 @@ def migrate_queues():
     return qs.update(queue=Val(TaskQueue.NET))
 
 
+def save_model(instance):
+    if 'sqlite' != db_vendor:
+        with atomic(durable=False):
+            instance.save()
+        return
+
+    # work around for SQLite and its many
+    # "database is locked" errors
+    with atomic(durable=False):
+        instance.save()
+    arg = getattr(settings, 'SQLITE_DELAY_FLOAT', 1.5)
+    time.sleep(random.expovariate(arg))
+
+
 @atomic(durable=False)
 def schedule_media_servers_update():
     # Schedule a task to update media servers
@@ -257,7 +276,7 @@ def index_source_task(source_id):
     '''
         Indexes media available from a Source object.
     '''
-    reset_queries()
+    db.reset_queries()
     cleanup_completed_tasks()
     # deleting expired media should happen any time an index task is requested
     cleanup_old_media()
@@ -272,7 +291,7 @@ def index_source_task(source_id):
     # Reset any errors
     # TODO: determine if this affects anything
     source.has_failed = False
-    source.save()
+    save_model(source)
     # Index the source
     videos = source.index_media()
     if not videos:
@@ -283,7 +302,7 @@ def index_source_task(source_id):
                                f'is reachable')
     # Got some media, update the last crawl timestamp
     source.last_crawl = timezone.now()
-    source.save()
+    save_model(source)
     num_videos = len(videos)
     log.info(f'Found {num_videos} media items for source: {source}')
     fields = lambda f, m: m.get_metadata_field(f)
@@ -490,7 +509,7 @@ def download_media_metadata(media_id):
         media.duration = media.metadata_duration
 
     # Don't filter media here, the post_save signal will handle that
-    media.save()
+    save_model(media)
     log.info(f'Saved {len(media.metadata)} bytes of metadata for: '
              f'{source} / {media}: {media_id}')
 
@@ -648,7 +667,7 @@ def download_media(media_id):
                 media.downloaded_hdr = cformat['is_hdr']
             else:
                 media.downloaded_format = 'audio'
-        media.save()
+        save_model(media)
         # If selected, copy the thumbnail over as well
         if media.source.copy_thumbnails:
             if not media.thumb_file_exists:
@@ -697,7 +716,7 @@ def save_all_media_for_source(source_id):
         source has its parameters changed and all media needs to be
         checked to see if its download status has changed.
     '''
-    reset_queries()
+    db.reset_queries()
     try:
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist as e:
@@ -742,14 +761,21 @@ def save_all_media_for_source(source_id):
         )
         saved_later.add(media.uuid)
 
+    # Keep out of the way of the index task!
+    # SQLite will be locked for a while if we start
+    # a large source, which reschedules a more costly task.
+    if 'sqlite' == db_vendor:
+        index_task = get_source_index_task(source_id)
+        if index_task and index_task.locked_by_pid_running():
+            raise Exception(_('Indexing not completed'))
+
     # Trigger the post_save signal for each media item linked to this source as various
     # flags may need to be recalculated
     tvn_format = '2/{:,}' + f'/{save_qs.count():,}'
     for mn, media in enumerate(qs_gen(save_qs), start=1):
         if media.uuid not in saved_later:
             update_task_status(task, tvn_format.format(mn))
-            with atomic():
+            save_model(media)
-                media.save()
     # Reset task.verbose_name to the saved value
     update_task_status(task, None)
 
@@ -766,8 +792,7 @@ def refresh_formats(media_id):
         log.debug(f'Failed to refresh formats for: {media.source} / {media.key}: {e!s}')
         pass
     else:
-        with atomic():
+        save_model(media)
-            media.save()
 
 
 @background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.FS), remove_existing_tasks=True)
@@ -826,15 +851,14 @@ def wait_for_media_premiere(media_id):
         if media.published < now:
             media.manual_skip = False
             media.skip = False
-            # start the download tasks after save
+            # the download tasks start after the media is saved
         else:
             media.manual_skip = True
             media.title = _(f'Premieres in {hours(media.published - now)} hours')
             task = get_media_premiere_task(media_id)
             if task:
                 update_task_status(task, f'available in {hours(media.published - now)} hours')
-        with atomic():
+        save_model(media)
-            media.save()
 
 
 @background(schedule=dict(priority=1, run_at=90), queue=Val(TaskQueue.FS), remove_existing_tasks=False)

tubesync/tubesync/local_settings.py.container

@@ -59,6 +59,12 @@ else:
     }
     DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
 
+    # the argument to random.expovariate(),
+    # a larger value means less delay
+    # with too little delay, you may see
+    # more "database is locked" errors
+    SQLITE_DELAY_FLOAT = 5
+
 
 DEFAULT_THREADS = 1
 BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True)