Merge pull request #842 from tcely/delete-source

Improve deleting sources

.gitignore

@@ -7,6 +7,9 @@ __pycache__/
 # C extensions
 *.so
 
+# vim swap files
+.*.swp
+
 # Distribution / packaging
 .Python
 build/

tubesync/sync/management/commands/delete-source.py

@@ -29,11 +29,12 @@ class Command(BaseCommand):
         except Source.DoesNotExist:
             raise CommandError(f'Source does not exist with '
                                f'UUID: {source_uuid}')
-        # Detach post-delete signal for Media so we don't spam media servers
-        signals.post_delete.disconnect(media_post_delete, sender=Media)
+        # Reconfigure the source to not update the disk or media servers
+        source.deactivate()
         # Delete the source, triggering pre-delete signals for each media item
         log.info(f'Found source with UUID "{source.uuid}" with name '
                  f'"{source.name}" and deleting it, this may take some time!')
+        log.info(f'Source directory: {source.directory_path}')
         source.delete()
         # Update any media servers
         for mediaserver in MediaServer.objects.all():
@@ -42,10 +43,9 @@ class Command(BaseCommand):
             rescan_media_server(
                 str(mediaserver.pk),
                 priority=0,
+                schedule=30,
                 verbose_name=verbose_name.format(mediaserver),
                 remove_existing_tasks=True
             )
-        # Re-attach signals
-        signals.post_delete.connect(media_post_delete, sender=Media)
         # All done
         log.info('Done')

tubesync/sync/models.py

@@ -333,6 +333,27 @@ class Source(models.Model):
         replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and')
         return slugify(replaced)[:80]
 
+    def deactivate(self):
+        self.download_media = False
+        self.index_streams = False
+        self.index_videos = False
+        self.index_schedule = IndexSchedule.NEVER
+        self.save(update_fields={
+            'download_media',
+            'index_streams',
+            'index_videos',
+            'index_schedule',
+        })
+
+    @property
+    def is_active(self):
+        active = (
+            self.download_media or
+            self.index_streams or
+            self.index_videos
+        )
+        return self.index_schedule and active
+
     @property
     def is_audio(self):
         return self.source_resolution == SourceResolution.AUDIO.value

tubesync/sync/signals.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from shutil import rmtree
 from tempfile import TemporaryDirectory
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
@@ -12,8 +13,8 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta
                     download_media_thumbnail, download_media_metadata,
                     map_task_to_instance, check_source_directory_exists,
                     download_media, rescan_media_server, download_source_images,
-                    save_all_media_for_source, rename_media,
-                    get_media_metadata_task, get_media_download_task)
+                    delete_all_media_for_source, save_all_media_for_source,
+                    rename_media, get_media_metadata_task, get_media_download_task)
 from .utils import delete_file, glob_quote, mkdir_p
 from .filtering import filter_media
 from .choices import Val, YouTube_SourceType
@@ -141,16 +142,34 @@ def source_post_save(sender, instance, created, **kwargs):
 def source_pre_delete(sender, instance, **kwargs):
     # Triggered before a source is deleted, delete all media objects to trigger
     # the Media models post_delete signal
-    for media in Media.objects.filter(source=instance):
-        log.info(f'Deleting media for source: {instance.name} item: {media.name}')
-        media.delete()
+    log.info(f'Deactivating source: {instance.name}')
+    instance.deactivate()
+    log.info(f'Deleting tasks for source: {instance.name}')
+    delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+    # Schedule deletion of media
+    verbose_name = _('Deleting all media for source "{}"')
+    delete_all_media_for_source(
+        str(instance.pk),
+        str(instance.name),
+        priority=1,
+        verbose_name=verbose_name.format(instance.name),
+    )
+    # Try to do it all immediately
+    # If this is killed, the scheduled task should do the work instead.
+    delete_all_media_for_source.now(
+        str(instance.pk),
+        str(instance.name),
+    )
 
 
 @receiver(post_delete, sender=Source)
 def source_post_delete(sender, instance, **kwargs):
     # Triggered after a source is deleted
-    log.info(f'Deleting tasks for source: {instance.name}')
-    delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+    source = instance
+    # Remove the directory, if the user requested that
+    directory_path = Path(source.directory_path)
+    if (directory_path / '.to_be_removed').is_file():
+        rmtree(directory_path, True)
 
 
 @receiver(task_failed, sender=Task)
@@ -344,6 +363,8 @@ def media_post_delete(sender, instance, **kwargs):
             log.info(f'Deleting file for: {instance} path: {file}')
             delete_file(file)
 
+    if not instance.source.is_active:
+        return
     # Schedule a task to update media servers
     for mediaserver in MediaServer.objects.all():
         log.info(f'Scheduling media server updates')

tubesync/sync/tasks.py

@@ -55,6 +55,7 @@ def map_task_to_instance(task):
         'sync.tasks.rename_media': Media,
         'sync.tasks.rename_all_media_for_source': Source,
         'sync.tasks.wait_for_media_premiere': Media,
+        'sync.tasks.delete_all_media_for_source': Source,
     }
     MODEL_URL_MAP = {
         Source: 'sync:source',
@@ -690,3 +691,23 @@ def wait_for_media_premiere(media_id):
         media.title = _(f'Premieres in {hours(media.published - now)} hours')
         media.save()
 
+@background(schedule=300, remove_existing_tasks=False)
+def delete_all_media_for_source(source_id, source_name):
+    source = None
+    try:
+        source = Source.objects.get(pk=source_id)
+    except Source.DoesNotExist:
+        # Task triggered but the source no longer exists, do nothing
+        log.error(f'Task delete_all_media_for_source(pk={source_id}) called but no '
+                  f'source exists with ID: {source_id}')
+        pass
+    mqs = Media.objects.all().defer(
+        'metadata',
+    ).filter(
+        source=source or source_id,
+    )
+    for media in mqs:
+        log.info(f'Deleting media for source: {source_name} item: {media.name}')
+        with atomic():
+            media.delete()
+

tubesync/sync/views.py

@@ -3,7 +3,6 @@ import os
 import json
 from base64 import b64decode
 import pathlib
-import shutil
 import sys
 from django.conf import settings
 from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
@@ -415,15 +414,8 @@ class DeleteSourceView(DeleteView, FormMixin):
         delete_media = True if delete_media_val is not False else False
         if delete_media:
             source = self.get_object()
-            for media in Media.objects.filter(source=source):
-                if media.media_file:
-                    file_path = media.media_file.path
-                    matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
-                    for file in matching_files:
-                        delete_file(file)
-            directory_path = source.directory_path
-            if os.path.exists(directory_path):
-                shutil.rmtree(directory_path, True)
+            directory_path = pathlib.Path(source.directory_path)
+            (directory_path / '.to_be_removed').touch(exist_ok=True)
         return super().post(request, *args, **kwargs)
 
     def get_success_url(self):