From 480d5dd8913286d34f66cf8d477f7820b279a90a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 11:38:32 -0500 Subject: [PATCH 01/13] Add `wait_for_media_premiere` task --- tubesync/sync/tasks.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d677df40..88dc576c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -52,6 +52,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, 'sync.tasks.rename_all_media_for_source': Source, + 'sync.tasks.wait_for_media_premiere': Media, } MODEL_URL_MAP = { Source: 'sync:source', @@ -126,6 +127,13 @@ def get_media_metadata_task(media_id): except IndexError: return False +def get_media_premiere_task(media_id): + try: + return Task.objects.get_task('sync.tasks.wait_for_media_premiere', + args=(str(media_id),))[0] + except IndexError: + return False + def delete_task_by_source(task_name, source_id): return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete() @@ -531,4 +539,29 @@ def rename_all_media_for_source(source_id): for media in Media.objects.filter(source=source): media.rename_files() +@background(repeat=3600, schedule=0) +def wait_for_media_premiere(media_id): + td = lambda p, now=timezone.now(): (p - now) + hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) + + try: + media = Media.objects.get(pk=media_id) + except Media.DoesNotExist: + return + if media.published < timezone.now(): + media.manual_skip = False + media.skip = False + # start the download tasks + media.save() + else: + media.manual_skip = True + media.title = _(f'Premieres in {hours(td(media.published))} hours') + task = get_media_premiere_task(str(media.pk)) + if task: + task.verbose_name = _(f'Waiting for premiere of "{media.key}" ' + f'in {hours(td(media.published))} hours') + if not task.repeat_until: + task.repeat_until = media.published + timedelta(hours=2) + task.save() + media.save() From 9e46b9f6d53533657aa0eb41d7d0ecd75150b1e1 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 12:29:36 -0500 Subject: [PATCH 02/13] A first attempt at waiting for future videos --- tubesync/sync/tasks.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 88dc576c..e9722f7f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -28,6 +28,7 @@ from .models import Source, Media, MediaServer from .utils import (get_remote_image, resize_image_to_height, delete_file, write_text_file, filter_response) from .filtering import filter_media +from .youtube import YouTubeError def get_hash(task_name, pk): @@ -312,7 +313,34 @@ def download_media_metadata(media_id): log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.') return source = media.source - metadata = media.index_metadata() + try: + metadata = media.index_metadata() + except YouTubeError as e: + e_str = str(e) + if ': Premieres in' in e_str: + published_datetime = None + now = timezone.now() + parts = e_str.rsplit(' ', 2) + try: + if 'hours' == parts[-1].lower(): + published_datetime = now + timedelta(hours=int(parts[-2], base=10)) + elif 'days' == parts[-1].lower(): + published_datetime = now + timedelta(days=int(parts[-2], base=10)) + except Exception as ee: + log.exception(ee) + pass + if published_datetime: + media.published = published_datetime + media.manual_skip = True + media.save() + wait_for_media_premiere( + str(media.pk), + priority=15, + queue=str(media.pk), + ) + log.exception(e) + log.debug(str(e)) + return response = metadata if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) From deeb2f61a973c30677b6c83307c61d99a3bf75f7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 13:00:30 -0500 Subject: [PATCH 03/13] `background` has less documentation than I wanted --- tubesync/sync/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index e9722f7f..8f1df2b7 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -337,6 +337,7 @@ def download_media_metadata(media_id): str(media.pk), priority=15, queue=str(media.pk), + repeat=Task.HOURLY, ) log.exception(e) log.debug(str(e)) @@ -567,7 +568,7 @@ def rename_all_media_for_source(source_id): for media in Media.objects.filter(source=source): media.rename_files() -@background(repeat=3600, schedule=0) +@background(schedule=0) def wait_for_media_premiere(media_id): td = lambda p, now=timezone.now(): (p - now) hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) From 89573e93f581ff4837a4b5ffbbbdc5285a27b1c2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 13:49:25 -0500 Subject: [PATCH 04/13] Add `verbose_name` for better display Also, some code readability changes. --- tubesync/sync/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 8f1df2b7..6d053811 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -318,26 +318,32 @@ def download_media_metadata(media_id): except YouTubeError as e: e_str = str(e) if ': Premieres in' in e_str: - published_datetime = None now = timezone.now() + published_datetime = None + parts = e_str.rsplit(' ', 2) + unit = lambda p: str(p)[-1].lower() + number = lambda p: int(str(p)[-2], base=10) try: - if 'hours' == parts[-1].lower(): - published_datetime = now + timedelta(hours=int(parts[-2], base=10)) - elif 'days' == parts[-1].lower(): - published_datetime = now + timedelta(days=int(parts[-2], base=10)) + if 'hours' == unit(parts): + published_datetime = now + timedelta(hours=number(parts)) + elif 'days' == unit(parts): + published_datetime = now + timedelta(days=number(parts)) except Exception as ee: log.exception(ee) pass + if published_datetime: media.published = published_datetime media.manual_skip = True media.save() + verbose_name = _('Waiting for the premiere of "{}"') wait_for_media_premiere( str(media.pk), priority=15, queue=str(media.pk), repeat=Task.HOURLY, + verbose_name=verbose_name.format(media.key), ) log.exception(e) log.debug(str(e)) From a028b4ea26899c101c37325f0a3b6b377783d4c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 14:09:53 -0500 Subject: [PATCH 05/13] Let the wait task go ahead first --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 6d053811..4831ca04 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -340,7 +340,7 @@ def download_media_metadata(media_id): verbose_name = _('Waiting for the premiere of "{}"') wait_for_media_premiere( str(media.pk), - priority=15, + priority=0, queue=str(media.pk), repeat=Task.HOURLY, verbose_name=verbose_name.format(media.key), From b05d3854a11b3d22b603d129602f48dd97d42100 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 2 Feb 2025 23:58:09 -0500 Subject: [PATCH 06/13] Use `repeat_until` in the call --- tubesync/sync/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4831ca04..1d850e7b 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -337,12 +337,13 @@ def download_media_metadata(media_id): media.published = published_datetime media.manual_skip = True media.save() - verbose_name = _('Waiting for the premiere of "{}"') + verbose_name = _('Waiting for the premiere of "{}" at: {published_datetime}') wait_for_media_premiere( str(media.pk), priority=0, queue=str(media.pk), repeat=Task.HOURLY, + repeat_until = published_datetime + timedelta(hours=2), verbose_name=verbose_name.format(media.key), ) log.exception(e) @@ -583,6 +584,8 @@ def wait_for_media_premiere(media_id): media = Media.objects.get(pk=media_id) except Media.DoesNotExist: return + if media.metadata: + return if media.published < timezone.now(): media.manual_skip = False media.skip = False @@ -595,8 +598,6 @@ def wait_for_media_premiere(media_id): if task: task.verbose_name = _(f'Waiting for premiere of "{media.key}" ' f'in {hours(td(media.published))} hours') - if not task.repeat_until: - task.repeat_until = media.published + timedelta(hours=2) task.save() media.save() From c0f52ae12eeba64300a27bc4e137cc5c7f4edf05 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 00:09:59 -0500 Subject: [PATCH 07/13] We only need one waiting task per video --- tubesync/sync/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1d850e7b..70bf6be2 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -345,6 +345,7 @@ def download_media_metadata(media_id): repeat=Task.HOURLY, repeat_until = published_datetime + timedelta(hours=2), verbose_name=verbose_name.format(media.key), + remove_existing_tasks=True, ) log.exception(e) log.debug(str(e)) From b50477e536684e22ea851cb561234e3fabc9a217 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 10:22:38 -0500 Subject: [PATCH 08/13] Add minutes as a unit also --- tubesync/sync/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 70bf6be2..4d2ecbc2 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -317,7 +317,7 @@ def download_media_metadata(media_id): metadata = media.index_metadata() except YouTubeError as e: e_str = str(e) - if ': Premieres in' in e_str: + if ': Premieres in ' in e_str: now = timezone.now() published_datetime = None @@ -325,7 +325,9 @@ def download_media_metadata(media_id): unit = lambda p: str(p)[-1].lower() number = lambda p: int(str(p)[-2], base=10) try: - if 'hours' == unit(parts): + if 'minutes' == unit(parts): + published_datetime = now + timedelta(minutes=number(parts)) + elif 'hours' == unit(parts): published_datetime = now + timedelta(hours=number(parts)) elif 'days' == unit(parts): published_datetime = now + timedelta(days=number(parts)) From bf0dcc631addc3f90d4f26ae7c469e759583ce78 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 11:33:56 -0500 Subject: [PATCH 09/13] Fixes from testing for `wait_for_media_premiere` --- tubesync/sync/tasks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4d2ecbc2..0433fe1b 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -317,13 +317,16 @@ def download_media_metadata(media_id): metadata = media.index_metadata() except YouTubeError as e: e_str = str(e) + log_exception = True if ': Premieres in ' in e_str: now = timezone.now() published_datetime = None - parts = e_str.rsplit(' ', 2) - unit = lambda p: str(p)[-1].lower() - number = lambda p: int(str(p)[-2], base=10) + parts = e_str.split(': ', 1)[1].rsplit(' ', 2) + unit = lambda p: str(p[-1]).lower() + number = lambda p: int(str(p[-2]), base=10) + log.debug(parts) + log.debug(unit(parts)) try: if 'minutes' == unit(parts): published_datetime = now + timedelta(minutes=number(parts)) @@ -339,17 +342,18 @@ def download_media_metadata(media_id): media.published = published_datetime media.manual_skip = True media.save() - verbose_name = _('Waiting for the premiere of "{}" at: {published_datetime}') + verbose_name = _('Waiting for the premiere of "{}" at: {}') wait_for_media_premiere( str(media.pk), priority=0, queue=str(media.pk), repeat=Task.HOURLY, repeat_until = published_datetime + timedelta(hours=2), - verbose_name=verbose_name.format(media.key), + verbose_name=verbose_name.format(media.key, published_datetime), remove_existing_tasks=True, ) - log.exception(e) + if log_exception: + log.exception(e) log.debug(str(e)) return response = metadata From a8e7299dbe655d0e9d6d255b5b377c21e8937244 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 11:44:02 -0500 Subject: [PATCH 10/13] Log units and number within the try block --- tubesync/sync/tasks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0433fe1b..276fb348 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -326,14 +326,15 @@ def download_media_metadata(media_id): unit = lambda p: str(p[-1]).lower() number = lambda p: int(str(p[-2]), base=10) log.debug(parts) - log.debug(unit(parts)) try: + if 'days' == unit(parts): + published_datetime = now + timedelta(days=number(parts)) + if 'hours' == unit(parts): + published_datetime = now + timedelta(hours=number(parts)) if 'minutes' == unit(parts): published_datetime = now + timedelta(minutes=number(parts)) - elif 'hours' == unit(parts): - published_datetime = now + timedelta(hours=number(parts)) - elif 'days' == unit(parts): - published_datetime = now + timedelta(days=number(parts)) + log.debug(unit(parts)) + log.debug(number(parts)) except Exception as ee: log.exception(ee) pass From df5746dac429351c93dd34e00c26b314d15c295c Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 11:52:25 -0500 Subject: [PATCH 11/13] Don't log the handled exception --- tubesync/sync/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 276fb348..3b149b4c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -353,6 +353,7 @@ def download_media_metadata(media_id): verbose_name=verbose_name.format(media.key, published_datetime), remove_existing_tasks=True, ) + log_exception = False if log_exception: log.exception(e) log.debug(str(e)) From aba3b1f16db73a5e2b1b0433672c26a0e05a792c Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 18:32:41 -0500 Subject: [PATCH 12/13] Display the `datetime` without partial seconds --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3b149b4c..58a4fb64 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -350,7 +350,7 @@ def download_media_metadata(media_id): queue=str(media.pk), repeat=Task.HOURLY, repeat_until = published_datetime + timedelta(hours=2), - verbose_name=verbose_name.format(media.key, published_datetime), + verbose_name=verbose_name.format(media.key, published_datetime.isoformat(' ', 'seconds')), remove_existing_tasks=True, ) log_exception = False From 5cf7f1574ab5084a435ac13993bff2abfde39188 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 3 Feb 2025 21:22:19 -0500 Subject: [PATCH 13/13] We don't need the extra run --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 58a4fb64..49ab9fe2 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -349,7 +349,7 @@ def download_media_metadata(media_id): priority=0, queue=str(media.pk), repeat=Task.HOURLY, - repeat_until = published_datetime + timedelta(hours=2), + repeat_until = published_datetime + timedelta(hours=1), verbose_name=verbose_name.format(media.key, published_datetime.isoformat(' ', 'seconds')), remove_existing_tasks=True, )