Merge pull request #824 from tcely/patch-15

Increase database efficiency

tubesync/sync/tasks.py

@@ -17,6 +17,7 @@ from django.conf import settings
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.utils import timezone
+from django.db.transaction import atomic
 from django.db.utils import IntegrityError
 from django.utils.translation import gettext_lazy as _
 from background_task import background
@@ -179,6 +180,7 @@ def cleanup_removed_media(source, videos):
 
 
 @background(schedule=300, remove_existing_tasks=True)
+@atomic(durable=True)
 def index_source_task(source_id):
     '''
         Indexes media available from a Source object.
@@ -221,7 +223,8 @@ def index_source_task(source_id):
         if published_dt is not None:
             media.published = published_dt
         try:
-            media.save()
+            with atomic():
+                media.save()
             log.debug(f'Indexed media: {source} / {media}')
             # log the new media instances
             new_media_instance = (
@@ -611,9 +614,10 @@ def save_all_media_for_source(source_id):
 
     # Trigger the post_save signal for each media item linked to this source as various
     # flags may need to be recalculated
-    for media in mqs:
+    with atomic():
-        if media.uuid not in already_saved:
+        for media in mqs:
-            media.save()
+            if media.uuid not in already_saved:
+                media.save()
 
 
 @background(schedule=60, remove_existing_tasks=True)
@@ -626,6 +630,7 @@ def rename_media(media_id):
 
 
 @background(schedule=300, remove_existing_tasks=True)
+@atomic(durable=True)
 def rename_all_media_for_source(source_id):
     try:
         source = Source.objects.get(pk=source_id)
@@ -653,7 +658,8 @@ def rename_all_media_for_source(source_id):
         downloaded=True,
     )
     for media in mqs:
-        media.rename_files()
+        with atomic():
+            media.rename_files()
 
 
 @background(schedule=60, remove_existing_tasks=True)

tubesync/sync/youtube.py

@@ -146,6 +146,14 @@ def get_media_info(url, days=None):
             f'yesterday-{days!s}days' if days else None
         )
     opts = get_yt_opts()
+    paths = opts.get('paths', dict())
+    if 'temp' in paths:
+        temp_dir_obj = TemporaryDirectory(prefix='.yt_dlp-', dir=paths['temp'])
+        temp_dir_path = Path(temp_dir_obj.name)
+        (temp_dir_path / '.ignore').touch(exist_ok=True)
+        paths.update({
+            'temp': str(temp_dir_path),
+        })
     opts.update({
         'ignoreerrors': False, # explicitly set this to catch exceptions
         'ignore_no_formats_error': False, # we must fail first to try again with this enabled
@@ -156,10 +164,14 @@ def get_media_info(url, days=None):
         'check_formats': True,
         'daterange': yt_dlp.utils.DateRange(start=start),
         'extractor_args': {
-            'youtube': {'formats': ['missing_pot']},
             'youtubetab': {'approximate_date': ['true']},
         },
+        'paths': paths,
+        'sleep_interval_requests': 2,
+        'verbose': True if settings.DEBUG else False,
     })
+    if start:
+        log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}')
     response = {}
     with yt_dlp.YoutubeDL(opts) as y:
         try: