From 2ebcaf42acf733ca1d426aea9efe3c963337683c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 18 Jun 2025 12:26:03 -0400 Subject: [PATCH 01/10] Provide a string where it is expected --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3870be81..ebc00091 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -992,7 +992,7 @@ def refresh_formats(media_id): retry=retry, ) # combine the strings - exc.args = (' '.join(exc.args),) + exc.args = (' '.join(map(str, exc.args)),) # store instance details exc.instance = dict( key=media.key, From b9d46ac36fd1f979423e8136540f72d8e8c203aa Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 18 Jun 2025 13:36:31 -0400 Subject: [PATCH 02/10] Add running tasks --- tubesync/common/huey.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index b030fe2e..47d6de17 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -26,7 +26,8 @@ def h_q_dict(q, /): return dict( scheduled=(q.scheduled_count(), q.scheduled(),), pending=(q.pending_count(), q.pending(),), - result=(q.result_count(), list(q.all_results().keys()),), + running=q._tasks_in_flight, + results=(q.result_count(), list(q.all_results().keys()),), ) From ea88c7beddabc724df362fd86fca9d5ccdc344fb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 18 Jun 2025 16:07:42 -0400 Subject: [PATCH 03/10] Lock each `Media` instance to prevent racing --- tubesync/sync/tasks.py | 59 ++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3870be81..5b159c3f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -27,7 +27,8 @@ from django.utils.translation import gettext_lazy as _ from background_task import background from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask -from django_huey import db_periodic_task, db_task, task as huey_task # noqa +from django_huey import lock_task as huey_lock_task, task as huey_task # noqa +from django_huey import db_periodic_task, db_task from huey import crontab as huey_crontab from common.huey import CancelExecution, dynamic_retry, register_huey_signals from common.logger import log @@ -401,22 +402,26 @@ def migrate_to_metadata(media_id): except Metadata.DoesNotExist as e: raise CancelExecution(_('no indexed data to migrate to metadata'), retry=False) from e - video = data.value - fields = lambda f, m: m.get_metadata_field(f) - timestamp = video.get(fields('timestamp', media), None) - for key in ('epoch', 'availability', 'extractor_key',): - field = fields(key, media) - value = video.get(field) - existing_value = media.get_metadata_first_value(key) - if value is None: - if 'epoch' == key: - value = timestamp - elif 'extractor_key' == key: - value = data.site - if value is not None: - if existing_value and ('epoch' == key or value == existing_value): - continue - media.save_to_metadata(field, value) + with huey_lock_task( + f'media:{media.uuid}', + queue=Val(TaskQueue.DB), + ): + video = data.value + fields = lambda f, m: m.get_metadata_field(f) + timestamp = video.get(fields('timestamp', media), None) + for key in ('epoch', 'availability', 'extractor_key',): + field = fields(key, media) + value = video.get(field) + existing_value = media.get_metadata_first_value(key) + if value is None: + if 'epoch' == key: + value = timestamp + elif 'extractor_key' == key: + value = data.site + if value is not None: + if existing_value and ('epoch' == key or value == existing_value): + continue + media.save_to_metadata(field, value) @background(schedule=dict(priority=0, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=False) @@ -1008,17 +1013,21 @@ def refresh_formats(media_id): @db_task(delay=60, priority=80, retries=5, retry_delay=60, queue=Val(TaskQueue.FS)) +@atomic(durable=True) def rename_media(media_id): try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: raise CancelExecution(_('no such media'), retry=False) from e else: - with atomic(): + with huey_lock_task( + f'media:{media.uuid}', + queue=Val(TaskQueue.DB), + ): media.rename_files() -@background(schedule=dict(priority=20, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=True) +@db_task(delay=300, priority=80, retries=5, retry_delay=600, queue=Val(TaskQueue.FS)) @atomic(durable=True) def rename_all_media_for_source(source_id): try: @@ -1027,7 +1036,7 @@ def rename_all_media_for_source(source_id): # Task triggered but the source no longer exists, do nothing log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') - raise InvalidTaskError(_('no such source')) from e + raise CancelExecution(_('no such source'), retry=False) from e # Check that the settings allow renaming rename_sources_setting = getattr(settings, 'RENAME_SOURCES') or list() create_rename_tasks = ( @@ -1038,14 +1047,18 @@ def rename_all_media_for_source(source_id): getattr(settings, 'RENAME_ALL_SOURCES', False) ) if not create_rename_tasks: - return + return None mqs = Media.objects.all().filter( source=source, downloaded=True, ) for media in qs_gen(mqs): - with atomic(): - media.rename_files() + with huey_lock_task( + f'media:{media.uuid}', + queue=Val(TaskQueue.DB), + ): + with atomic(): + media.rename_files() @background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.DB), remove_existing_tasks=True) From c2f7863a0f544557887b6fb90ac4de11cdc979eb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 18 Jun 2025 18:14:44 -0400 Subject: [PATCH 04/10] Show the targeted schedule on the sources page --- tubesync/sync/templates/sync/sources.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/sources.html b/tubesync/sync/templates/sync/sources.html index 0e766e23..0f13463b 100644 --- a/tubesync/sync/templates/sync/sources.html +++ b/tubesync/sync/templates/sync/sources.html @@ -31,7 +31,8 @@ {% if source.has_failed %} Source has permanent failures {% else %} - {{ source.media_count }} media items, {{ source.downloaded_count }} downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %} + {{ source.media_count }} media items, {{ source.downloaded_count }} downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
+ Next update target: {% if source.target_schedule %}{{ source.target_schedule|date:'l, h:00 A' }}{% else %}Not set{% endif %} {% endif %} Sync Now From 9f5f35ce1307f00843f99f70421ac197f536368f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 09:45:48 -0400 Subject: [PATCH 05/10] Add `allow_unicode=True` to `slugify` --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 851427b3..51fc89a2 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -723,7 +723,7 @@ class Media(models.Model): @property def slugtitle(self): replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and') - return slugify(replaced)[:80] + return slugify(replaced, allow_unicode=True)[:80] @property def thumbnail(self): From d26a820e17970f759ec1c290e59e4d367b3c9692 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 10:15:52 -0400 Subject: [PATCH 06/10] Round-trip through the encoder like `slugify` does for `ASCII` --- tubesync/sync/models/media.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 51fc89a2..53242d37 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -722,8 +722,11 @@ class Media(models.Model): @property def slugtitle(self): - replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and') - return slugify(replaced, allow_unicode=True)[:80] + no_underscores = self.title.replace('_', '-') + to_and = no_underscores.replace('&', 'and').replace('+', 'and') + slugified = slugify(to_and, allow_unicode=True) + decoded = slugified.encode(errors=''ignore').decode() + return decoded[:80] @property def thumbnail(self): From a016c8a9c9712ba8acccadfdd6517abdb0431310 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 10:24:35 -0400 Subject: [PATCH 07/10] Use `str.translate` --- tubesync/sync/models/media.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 53242d37..3cee7f6c 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -722,9 +722,11 @@ class Media(models.Model): @property def slugtitle(self): - no_underscores = self.title.replace('_', '-') - to_and = no_underscores.replace('&', 'and').replace('+', 'and') - slugified = slugify(to_and, allow_unicode=True) + transtab = str.maketrans({ + '_': '-', + '&': 'and', '+': 'and', + }) + slugified = slugify(self.title.translate(transtab), allow_unicode=True) decoded = slugified.encode(errors=''ignore').decode() return decoded[:80] From 5ccd50799d26c34d870fff899eec486c82b62eb8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 10:27:11 -0400 Subject: [PATCH 08/10] fixup: formatting and syntax --- tubesync/sync/models/media.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 3cee7f6c..b4726cd8 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -726,8 +726,11 @@ class Media(models.Model): '_': '-', '&': 'and', '+': 'and', }) - slugified = slugify(self.title.translate(transtab), allow_unicode=True) - decoded = slugified.encode(errors=''ignore').decode() + slugified = slugify( + self.title.translate(transtab), + allow_unicode=True, + ) + decoded = slugified.encode(errors='ignore').decode() return decoded[:80] @property From cb08f475d040c645089022d31b16c1bc13a3df23 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 10:48:05 -0400 Subject: [PATCH 09/10] Specify the `encoding` also --- tubesync/sync/models/media.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index b4726cd8..2f64b5f6 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -730,7 +730,11 @@ class Media(models.Model): self.title.translate(transtab), allow_unicode=True, ) - decoded = slugified.encode(errors='ignore').decode() + encoding = os.sys.getfilesystemencoding() + decoded = slugified.encode( + encoding=encoding, + errors='ignore', + ).decode(encoding=encoding) return decoded[:80] @property From f4f345c656303ce50f849c8391e6d88672e5542a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 19 Jun 2025 10:54:08 -0400 Subject: [PATCH 10/10] Allow underscores --- tubesync/sync/models/media.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 2f64b5f6..60fcf87f 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -723,7 +723,6 @@ class Media(models.Model): @property def slugtitle(self): transtab = str.maketrans({ - '_': '-', '&': 'and', '+': 'and', }) slugified = slugify(