From 3bc9310c98a47521ca6e01e79dd15b621d3df23e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 03:48:38 -0400 Subject: [PATCH 01/13] Log more human-friendly output --- tubesync/sync/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 466e41d3..bf5e43ed 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -35,7 +35,7 @@ from common.utils import ( django_queryset_generator as qs_gen, from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, - write_text_file, filter_response, ) + write_text_file, filter_response, seconds_to_timestr, ) from .youtube import YouTubeError db_vendor = db.connection.vendor @@ -256,8 +256,11 @@ def wait_for_errors(model, /, *, task_name=None): ) for task in tasks: update_task_status(task, 'paused (429)') - log.info(f'waiting for errors: 429 ({tqs.count()}): {model}') - time.sleep(10 * tqs.count()) + + delay = 10 * tqs.count() + time_str = seconds_to_timestr(delay) + log.info(f'waiting for errors: 429 ({time_str}): {model}') + time.sleep(delay) for task in tasks: update_task_status(task, None) From d4eeb776e829ee078a76aa0f13e61d9da8e0bee7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:12:35 -0400 Subject: [PATCH 02/13] Close the `tr` tag on the next line --- tubesync/sync/templates/sync/media-item.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html index b70f78c2..074987ea 100644 --- a/tubesync/sync/templates/sync/media-item.html +++ b/tubesync/sync/templates/sync/media-item.html @@ -121,7 +121,8 @@ {% if media_file_path == media.filepath %}  (matched) {% endif %} - + + File size File size
{{ media.downloaded_filesize|bytesformat }} From 1415af50015d2cb716548463a6e78475e9b426e4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:53:36 -0400 Subject: [PATCH 03/13] Allow skipping the loop --- tubesync/sync/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 5bc90d25..762e29fe 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,10 +130,10 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path): +def directory_and_stem(arg_path, /, *, only_once=False): filepath = Path(arg_path) stem = Path(filepath.stem) - while stem.suffixes and '' != stem.suffix: + while not only_once and stem.suffixes and '' != stem.suffix: stem = Path(stem.stem) stem = str(stem) return (filepath.parent, stem,) From 54aa9fd165d522eae32888943e4c820935d2c962 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:57:33 -0400 Subject: [PATCH 04/13] The video file should not have multiple extensions --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 11b30f80..4f4a16aa 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1172,7 +1172,7 @@ class Media(models.Model): self.save(update_fields=('media_file', 'skip')) log.info(f'Updated "media_file" in the database for: {self!s}') - (new_prefix_path, new_stem) = directory_and_stem(new_video_path) + (new_prefix_path, new_stem) = directory_and_stem(new_video_path, only_once=True) # move and change names to match stem for other_path in other_paths: From 15898df3bbb0205d97a9070ef3d0486e9bb997b6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:03:08 -0400 Subject: [PATCH 05/13] The old video file should also have a single extension --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 4f4a16aa..cad249a3 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1152,7 +1152,7 @@ class Media(models.Model): # collect the list of files to move # this should not include the video we just moved - (old_prefix_path, old_stem) = directory_and_stem(old_video_path) + (old_prefix_path, old_stem) = directory_and_stem(old_video_path, only_once=True) other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) log.info(f'Collected {len(other_paths)} other paths for: {self!s}') From fc3a68393d946e2369c30173f09feddb903702ea Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:08:53 -0400 Subject: [PATCH 06/13] Loop for the fuzzy stem --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index cad249a3..454d82ee 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1193,7 +1193,7 @@ class Media(models.Model): other_path.replace(new_file_path) for fuzzy_path in fuzzy_paths: - (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path) + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, only_once=False) old_file_str = fuzzy_path.name new_file_str = new_stem + old_file_str[len(fuzzy_stem):] new_file_path = Path(new_prefix_path / new_file_str) From 5c2dfca00f4b314e0c824e0727ae9060a80e2ddc Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:10:07 -0400 Subject: [PATCH 07/13] Change the default to not loop --- tubesync/sync/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 762e29fe..6cc2e0d3 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,7 +130,7 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path, /, *, only_once=False): +def directory_and_stem(arg_path, /, *, only_once=True): filepath = Path(arg_path) stem = Path(filepath.stem) while not only_once and stem.suffixes and '' != stem.suffix: From 24105db5cb243a2e5956c6821250b7623f7d16cf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:49:21 -0400 Subject: [PATCH 08/13] Use private `directory_and_stem` function --- tubesync/sync/models/_private.py | 8 ++++++++ tubesync/sync/models/media.py | 11 +++++------ tubesync/sync/utils.py | 9 --------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 5ec14d7c..baca9e11 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -10,3 +10,11 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2 element.tail = tail + (char * indent) return element +def directory_and_stem(arg_path, /, all_suffixes=False): + filepath = Path(arg_path) + stem = Path(filepath.stem) + while all_suffixes and stem.suffixes and '' != stem.suffix: + stem = Path(stem.stem) + stem = str(stem) + return (filepath.parent, stem,) + diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 454d82ee..62f73d5d 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -25,8 +25,7 @@ from ..youtube import ( ) from ..utils import ( seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote, - multi_key_sort, + write_text_file, mkdir_p, glob_quote, multi_key_sort, ) from ..matching import ( get_best_combined_format, @@ -39,7 +38,7 @@ from ..choices import ( from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) -from ._private import _srctype_dict, _nfo_element +from ._private import _srctype_dict, _nfo_element, directory_and_stem from .media__tasks import ( download_checklist, download_finished, wait_for_premiere, ) @@ -1152,7 +1151,7 @@ class Media(models.Model): # collect the list of files to move # this should not include the video we just moved - (old_prefix_path, old_stem) = directory_and_stem(old_video_path, only_once=True) + (old_prefix_path, old_stem) = directory_and_stem(old_video_path) other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) log.info(f'Collected {len(other_paths)} other paths for: {self!s}') @@ -1172,7 +1171,7 @@ class Media(models.Model): self.save(update_fields=('media_file', 'skip')) log.info(f'Updated "media_file" in the database for: {self!s}') - (new_prefix_path, new_stem) = directory_and_stem(new_video_path, only_once=True) + (new_prefix_path, new_stem) = directory_and_stem(new_video_path) # move and change names to match stem for other_path in other_paths: @@ -1193,7 +1192,7 @@ class Media(models.Model): other_path.replace(new_file_path) for fuzzy_path in fuzzy_paths: - (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, only_once=False) + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, True) old_file_str = fuzzy_path.name new_file_str = new_stem + old_file_str[len(fuzzy_stem):] new_file_path = Path(new_prefix_path / new_file_str) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 6cc2e0d3..fc7874fd 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,15 +130,6 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path, /, *, only_once=True): - filepath = Path(arg_path) - stem = Path(filepath.stem) - while not only_once and stem.suffixes and '' != stem.suffix: - stem = Path(stem.stem) - stem = str(stem) - return (filepath.parent, stem,) - - def mkdir_p(arg_path, mode=0o777): ''' Reminder: mode only affects the last directory From 08800a0c71d35ac2fd59a97af44f26ce2b348e70 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:52:30 -0400 Subject: [PATCH 09/13] fixup: import Path --- tubesync/sync/models/_private.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index baca9e11..8cf41ce1 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,3 +1,4 @@ +from pathlib import Path from ..choices import Val, YouTube_SourceType # noqa From 00f2795b086486df8906ca0932758c6f084d3142 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:55:40 -0400 Subject: [PATCH 10/13] Fail when `ruff` finds errors --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22bd31b2..5ae58c0a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -144,7 +144,7 @@ jobs: done } >> "${GITHUB_STEP_SUMMARY}" uvx --no-config --no-managed-python --no-progress --isolated \ - ruff check --exit-zero \ + ruff check \ --target-version "${target_version}" \ --output-format github \ --ignore "${ignore_csv_list}" From b0721fea80a7c65def8450e45c749fa589fadc17 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 01:03:30 -0400 Subject: [PATCH 11/13] Tighten `errors_qs` by checking `last_error` --- tubesync/sync/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 4f683834..093cad3f 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -845,9 +845,11 @@ class TasksView(ListView): data = super().get_context_data(*args, **kwargs) now = timezone.now() qs = Task.objects.all() - errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True) running_qs = qs.filter(locked_by__isnull=False) scheduled_qs = qs.filter(locked_by__isnull=True) + errors_qs = scheduled_qs.filter( + attempts__gt=0 + ).exclude(last_error__exact='') # Add to context data from ListView data['message'] = self.message From 3e310aa9f94eb3a2399165070d54d9756f958267 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 10:04:50 -0400 Subject: [PATCH 12/13] `yt-dlp-get-pot` was recently superceded by a framework in `yt-dlp` --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index 2e3f3acf..49b5127b 100644 --- a/Pipfile +++ b/Pipfile @@ -24,5 +24,4 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"} emoji = "*" brotli = "*" html5lib = "*" -yt-dlp-get-pot = "*" bgutil-ytdlp-pot-provider = "*" From 0830812e2133e99a1adcbc27f53777b25c469967 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 10:28:56 -0400 Subject: [PATCH 13/13] Bump the version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 3ab7f9ff..cd6d1dca 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -8,7 +8,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.3' +VERSION = '0.15.4' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = []