From 5a4379391b284919b7298cb193211c5c68ad0159 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:18:35 -0400 Subject: [PATCH 1/8] Migrate `check_source_directory_exists` to `huey` --- tubesync/sync/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 97a63515..51605c2d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -61,7 +61,6 @@ def map_task_to_instance(task): TASK_MAP = { 'sync.tasks.migrate_to_metadata': Media, 'sync.tasks.index_source_task': Source, - 'sync.tasks.check_source_directory_exists': Source, 'sync.tasks.download_media_thumbnail': Media, 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, @@ -536,7 +535,7 @@ def index_source_task(source_id): ) -@background(schedule=dict(priority=0, run_at=0), queue=Val(TaskQueue.FS)) +@dynamic_retry(db_task, priority=100, retries=15, queue=Val(TaskQueue.FS)) def check_source_directory_exists(source_id): ''' Checks the output directory for a source exists and is writable, if it does @@ -547,7 +546,7 @@ def check_source_directory_exists(source_id): source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: # Task triggered but the Source has been deleted, delete the task - raise InvalidTaskError(_('no such source')) from e + raise CancelExecution(_('no such source'), retry=False) from e # Check the source output directory exists if not source.directory_exists(): # Try to create it From 2422d9e56c3b61dfedf786b5883384633fa92ef2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:22:09 -0400 Subject: [PATCH 2/8] Update reset-tasks.py --- tubesync/sync/management/commands/reset-tasks.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index ae38a464..c7242114 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -18,11 +18,7 @@ class Command(BaseCommand): Task.objects.all().delete() # Iter all sources, creating new tasks for source in Source.objects.all(): - verbose_name = _('Check download directory exists for source "{}"') - check_source_directory_exists( - str(source.pk), - verbose_name=verbose_name.format(source.name), - ) + check_source_directory_exists(str(source.pk)) # Recreate the initial indexing task log.info(f'Resetting tasks for source: {source}') verbose_name = _('Index media from source "{}"') From a9701c5a5cb6781b90c14acbac76d6f1c56f7efe Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:24:00 -0400 Subject: [PATCH 3/8] Update tests.py --- tubesync/sync/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index bf1e4932..afccae4a 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -212,7 +212,7 @@ class FrontEndTestCase(TestCase): args=(source_uuid,))[0] self.assertEqual(task.queue, Val(TaskQueue.NET)) # Run the check_source_directory_exists task - check_source_directory_exists.now(source_uuid) + check_source_directory_exists.call_local(source_uuid) # Check the source is now on the source overview page response = c.get('/sources') self.assertEqual(response.status_code, 200) From c4d8928029a52bee765c6bdb3fd17b642642cebb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:29:01 -0400 Subject: [PATCH 4/8] Update signals.py --- tubesync/sync/signals.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 998ab3a3..07577aeb 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -34,7 +34,7 @@ def source_pre_save(sender, instance, **kwargs): return args = ( str(instance.pk), ) - check_source_directory_exists.now(*args) + check_source_directory_exists.call_local(*args) existing_dirpath = existing_source.directory_path.resolve(strict=True) new_dirpath = instance.directory_path.resolve(strict=False) if existing_dirpath != new_dirpath: @@ -105,11 +105,7 @@ def source_pre_save(sender, instance, **kwargs): def source_post_save(sender, instance, created, **kwargs): # Check directory exists and create an indexing task for newly created sources if created: - verbose_name = _('Check download directory exists for source "{}"') - check_source_directory_exists( - str(instance.pk), - verbose_name=verbose_name.format(instance.name), - ) + check_source_directory_exists(str(instance.pk)) if instance.source_type != Val(YouTube_SourceType.PLAYLIST) and instance.copy_channel_images: download_source_images(str(instance.pk)) if instance.index_schedule > 0: @@ -139,7 +135,6 @@ def source_pre_delete(sender, instance, **kwargs): instance.deactivate() log.info(f'Deleting tasks for source: {instance.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) - delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk) delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) @@ -166,7 +161,6 @@ def source_post_delete(sender, instance, **kwargs): source = instance log.info(f'Deleting tasks for removed source: {source.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) - delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk) delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) From bbf68517af163835d12fa26342bf31430a63811a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:31:41 -0400 Subject: [PATCH 5/8] Update views.py --- tubesync/sync/views.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 493098cd..8ee0f29a 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -994,11 +994,7 @@ class ResetTasks(FormView): Task.objects.all().delete() # Iter all tasks for source in Source.objects.all(): - verbose_name = _('Check download directory exists for source "{}"') - check_source_directory_exists( - str(source.pk), - verbose_name=verbose_name.format(source.name), - ) + check_source_directory_exists(str(source.pk)) # Recreate the initial indexing task verbose_name = _('Index media from source "{}"') index_source_task( From f6b589bbb048b48b4cbc8d70beb637cc12e9253c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 14:48:20 -0400 Subject: [PATCH 6/8] Log slightly more information when debugging --- tubesync/tubesync/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index f5689dd1..0e95d47a 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -57,12 +57,16 @@ DJANGO_HUEY = { 'limited': sqlite_tasks('limited', prefix='net'), 'network': sqlite_tasks('network'), }, + 'verbose': None if 'true' == getenv('TUBESYNC_DEBUG', False).strip().lower() else False, } for django_huey_queue in DJANGO_HUEY['queues'].values(): connection = django_huey_queue.get('connection') if connection: filepath = Path('/.' + connection.get('filename') or '').resolve(strict=False) filepath.parent.mkdir(exist_ok=True, parents=True) + consumer = django_huey_queue.get('consumer') + if consumer: + consumer['verbose'] = DJANGO_HUEY.get('verbose', False) TEMPLATES = [ From d43b691673343ec67ad1a50199950643b51d8027 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 15:19:57 -0400 Subject: [PATCH 7/8] fixup: remove an unused import --- tubesync/sync/management/commands/reset-tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index c7242114..398e49f4 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -1,6 +1,5 @@ from django.core.management.base import BaseCommand, CommandError # noqa from django.db.transaction import atomic -from django.utils.translation import gettext_lazy as _ from background_task.models import Task from common.logger import log from sync.models import Source From b5e6b41449de0cf031dbb5db216ed53c161fb8f2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 15 Jun 2025 15:22:40 -0400 Subject: [PATCH 8/8] fixup: it is only unused when everything is merged --- tubesync/sync/management/commands/reset-tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index 398e49f4..050eb448 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -1,5 +1,6 @@ from django.core.management.base import BaseCommand, CommandError # noqa from django.db.transaction import atomic +from django.utils.translation import gettext_lazy as _ # noqa from background_task.models import Task from common.logger import log from sync.models import Source