From 0b92ae0500aa1675c7ff26f606faa7783d6f8ecc Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 6 Apr 2025 18:42:07 -0400 Subject: [PATCH] Assign task queues based on resources used --- tubesync/sync/tasks.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index b539a2f9..aec69216 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -229,7 +229,7 @@ def cleanup_removed_media(source, videos): schedule_media_servers_update() -@background(schedule=dict(priority=10, run_at=30), remove_existing_tasks=True) +@background(schedule=dict(priority=10, run_at=30), queue='network', remove_existing_tasks=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -316,7 +316,7 @@ def index_source_task(source_id): cleanup_removed_media(source, videos) -@background(schedule=dict(priority=0, run_at=0)) +@background(schedule=dict(priority=0, run_at=0), queue='filesystem') def check_source_directory_exists(source_id): ''' Checks the output directory for a source exists and is writable, if it does @@ -335,7 +335,7 @@ def check_source_directory_exists(source_id): source.make_directory() -@background(schedule=dict(priority=5, run_at=10)) +@background(schedule=dict(priority=5, run_at=10), queue='network') def download_source_images(source_id): ''' Downloads an image and save it as a local thumbnail attached to a @@ -385,7 +385,7 @@ def download_source_images(source_id): log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}') -@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True) +@background(schedule=dict(priority=20, run_at=60), queue='network', remove_existing_tasks=True) def download_media_metadata(media_id): ''' Downloads the metadata for a media item. @@ -472,7 +472,7 @@ def download_media_metadata(media_id): f'{source} / {media}: {media_id}') -@background(schedule=dict(priority=15, run_at=10), remove_existing_tasks=True) +@background(schedule=dict(priority=15, run_at=10), queue='network', remove_existing_tasks=True) def download_media_thumbnail(media_id, url): ''' Downloads an image from a URL and save it as a local thumbnail attached to a @@ -510,7 +510,7 @@ def download_media_thumbnail(media_id, url): return True -@background(schedule=dict(priority=15, run_at=60), remove_existing_tasks=True) +@background(schedule=dict(priority=15, run_at=60), queue='network', remove_existing_tasks=True) def download_media(media_id): ''' Downloads the media to disk and attaches it to the Media instance. @@ -632,7 +632,7 @@ def download_media(media_id): raise DownloadFailedException(err) -@background(schedule=dict(priority=0, run_at=30), remove_existing_tasks=True) +@background(schedule=dict(priority=0, run_at=30), queue='network', remove_existing_tasks=True) def rescan_media_server(mediaserver_id): ''' Attempts to request a media rescan on a remote media server. @@ -647,7 +647,7 @@ def rescan_media_server(mediaserver_id): mediaserver.update() -@background(schedule=dict(priority=25, run_at=600), remove_existing_tasks=True) +@background(schedule=dict(priority=25, run_at=600), queue='network', remove_existing_tasks=True) def save_all_media_for_source(source_id): ''' Iterates all media items linked to a source and saves them to @@ -704,7 +704,7 @@ def save_all_media_for_source(source_id): update_task_status(task, None) -@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True) +@background(schedule=dict(priority=20, run_at=60), queue='filesystem', remove_existing_tasks=True) def rename_media(media_id): try: media = Media.objects.defer('metadata', 'thumb').get(pk=media_id) @@ -713,7 +713,7 @@ def rename_media(media_id): media.rename_files() -@background(schedule=dict(priority=20, run_at=300), remove_existing_tasks=True) +@background(schedule=dict(priority=20, run_at=300), queue='filesystem', remove_existing_tasks=True) @atomic(durable=True) def rename_all_media_for_source(source_id): try: @@ -746,7 +746,7 @@ def rename_all_media_for_source(source_id): media.rename_files() -@background(schedule=dict(priority=0, run_at=60), remove_existing_tasks=True) +@background(schedule=dict(priority=0, run_at=60), queue='database', remove_existing_tasks=True) def wait_for_media_premiere(media_id): hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) @@ -770,7 +770,7 @@ def wait_for_media_premiere(media_id): if task: update_task_status(task, f'available in {hours(media.published - now)} hours') -@background(schedule=dict(priority=1, run_at=300), remove_existing_tasks=False) +@background(schedule=dict(priority=1, run_at=300), queue='filesystem', remove_existing_tasks=False) def delete_all_media_for_source(source_id, source_name): source = None try: