From f724c5765ff595993ea86017f2268338dad8c0c0 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 14:52:09 -0400 Subject: [PATCH 001/174] Do not cache the blank thumbnail for so long --- tubesync/sync/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index a23597ce..73fb3ddd 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -523,6 +523,9 @@ class MediaThumbView(DetailView): def get(self, request, *args, **kwargs): media = self.get_object() + # Thumbnail media is never updated so we can ask the browser to cache it + # for ages, 604800 = 7 days + max_age = 604800 if media.thumb_file_exists: thumb_path = pathlib.Path(media.thumb.path) thumb = thumb_path.read_bytes() @@ -532,10 +535,10 @@ class MediaThumbView(DetailView): thumb = b64decode('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAA' 'AAAABAAEAAAICTAEAOw==') content_type = 'image/gif' + max_age = 600 response = HttpResponse(thumb, content_type=content_type) - # Thumbnail media is never updated so we can ask the browser to cache it - # for ages, 604800 = 7 days - response['Cache-Control'] = 'public, max-age=604800' + + response['Cache-Control'] = f'public, max-age={max_age}' return response From f9bb33c3abddc671517f7e45ede5e2209c394fd3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 17:43:35 -0400 Subject: [PATCH 002/174] Save thumbnail when refreshing formats --- tubesync/sync/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a68e1c4b..eec9d7ca 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -25,7 +25,8 @@ from .youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) from .utils import (seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote) + write_text_file, mkdir_p, directory_and_stem, glob_quote, + multi_key_sort) from .matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) from .fields import CommaSepChoiceField @@ -1268,6 +1269,27 @@ class Media(models.Model): if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) + # save the new list of thumbnails + thumbnails = self.get_metadata_first_value( + 'thumbnails', + self.get_metadata_first_value('thumbnails', []), + arg_dict=response, + ) + field = self.get_metadata_field('thumbnails') + self.save_to_metadata(field, thumbnails) + + # select and save our best thumbnail url + try: + thumbnail = [ thumb.get('url') for thumb in multi_key_sort( + thumbnails, + [('preference', True,)], + ) if thumb.get('url', '').endswith('.jpg') ][0] + except IndexError: + pass + else: + field = self.get_metadata_field('thumbnail') + self.save_to_metadata(field, thumbnail) + field = self.get_metadata_field('formats') self.save_to_metadata(field, response.get(field, [])) self.save_to_metadata(refreshed_key, response.get('epoch', formats_seconds)) From 6fe76eb34aa5ae294caf64b1fe95f6a519fb6136 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 15:54:21 -0400 Subject: [PATCH 003/174] Adjust metadata & media download delays --- tubesync/tubesync/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index dac5896f..9960f60e 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -135,7 +135,7 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) MAX_ATTEMPTS = 15 # Number of times tasks will be retried -MAX_RUN_TIME = 1*(24*60*60) # Maximum amount of time in seconds a task can run +MAX_RUN_TIME = 12*(60*60) # Maximum amount of time in seconds a task can run BACKGROUND_TASK_RUN_ASYNC = False # Run tasks async in the background BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once MAX_BACKGROUND_TASK_ASYNC_THREADS = 8 # For sanity reasons @@ -173,6 +173,8 @@ YOUTUBE_DEFAULTS = { 'cachedir': False, # Disable on-disk caching 'addmetadata': True, # Embed metadata during postprocessing where available 'geo_verification_proxy': getenv('geo_verification_proxy').strip() or None, + 'max_sleep_interval': (60)*5, + 'sleep_interval': 0.25, } COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' @@ -210,7 +212,7 @@ except: if MAX_RUN_TIME < 600: MAX_RUN_TIME = 600 -DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 50) +DOWNLOAD_MEDIA_DELAY = 1 + round(MAX_RUN_TIME / 100) if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS From 46dc2c9fee157bacc035c8d7cafd1aae94090f51 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 16:35:38 -0400 Subject: [PATCH 004/174] Include `missing_pot` formats in testing --- tubesync/sync/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index ffcbb074..55046c81 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -198,6 +198,7 @@ def get_media_info(url, /, *, days=None, info_json=None): 'clean_infojson': False, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { + 'youtube': {'formats': ['missing_pot']}, 'youtubetab': {'approximate_date': ['true']}, }, 'outtmpl': outtmpl, From 3f073b3ec336ba07391059420e915460db30a077 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 19:09:55 -0400 Subject: [PATCH 005/174] Include openresty --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c21a18c1..3d1d55ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,28 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -f /var/cache/debconf/*.dat-old +FROM alpine:${ALPINE_VERSION} AS openresty-debian +ARG TARGETARCH +ARG DEBIAN_VERSION +ADD 'https://openresty.org/package/pubkey.gpg' '/downloaded/pubkey.gpg' +RUN set -eu ; \ + decide_arch() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '' ;; \ + (arm64) printf -- 'arm64/' ;; \ + esac ; \ + } ; \ + set -x ; \ + mkdir -v -p '/etc/apt/trusted.gpg.d' && \ + apk --no-cache --no-progress add cmd:gpg2 && \ + gpg2 --dearmor \ + -o '/etc/apt/trusted.gpg.d/openresty.gpg' \ + < '/downloaded/pubkey.gpg' && \ + mkdir -v -p '/etc/apt/sources.list.d' && \ + printf -- >| '/etc/apt/sources.list.d/openresty.list' \ + 'deb http://openresty.org/package/%sdebian %s openresty' \ + "$(decide_arch)" "${DEBIAN_VERSION%-slim}" + FROM alpine:${ALPINE_VERSION} AS ffmpeg-download ARG FFMPEG_DATE ARG FFMPEG_VERSION @@ -257,7 +279,36 @@ RUN set -eu ; \ FROM scratch AS s6-overlay COPY --from=s6-overlay-extracted /s6-overlay-rootfs / -FROM tubesync-base AS tubesync +FROM tubesync-base AS tubesync-openresty + +COPY --from=openresty-debian \ + /etc/apt/trusted.gpg.d/openresty.gpg /etc/apt/trusted.gpg.d/openresty.gpg +COPY --from=openresty-debian \ + /etc/apt/sources.list.d/openresty.list /etc/apt/sources.list.d/openresty.list + +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -x && \ + apt-get update && \ + apt-get -y --no-install-recommends install nginx-light openresty && \ + # Clean up + apt-get -y autopurge && \ + apt-get -y autoclean && \ + rm -v -f /var/cache/debconf/*.dat-old + +FROM tubesync-base AS tubesync-nginx + +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -x && \ + apt-get update && \ + apt-get -y --no-install-recommends install nginx-light && \ + # Clean up + apt-get -y autopurge && \ + apt-get -y autoclean && \ + rm -v -f /var/cache/debconf/*.dat-old + +FROM tubesync-openresty AS tubesync ARG S6_VERSION @@ -282,7 +333,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va libmariadb3 \ libpq5 \ libwebp7 \ - nginx-light \ pipenv \ pkgconf \ python3 \ From a8d8e9d055ef4d2e1e43a7eeae557a0b93df5334 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:23:14 -0400 Subject: [PATCH 006/174] Update run --- config/root/etc/s6-overlay/s6-rc.d/nginx/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index 87769e62..63653343 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -2,4 +2,4 @@ cd / -exec /usr/sbin/nginx +exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr From cfb4b4ca1717517b9a5c1f33a7d25747d9445138 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:32:54 -0400 Subject: [PATCH 007/174] Don't install `nginx` binaries --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3d1d55ee..c3a1c5a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -290,7 +290,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ set -x && \ apt-get update && \ - apt-get -y --no-install-recommends install nginx-light openresty && \ + apt-get -y --no-install-recommends install nginx-common openresty && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ From 7ca79af746dc65c345cbadb8c81bf21e12e247aa Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:36:20 -0400 Subject: [PATCH 008/174] Use the `openresty` binary --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c3a1c5a1..64782388 100644 --- a/Dockerfile +++ b/Dockerfile @@ -456,7 +456,8 @@ RUN set -x && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ # Check nginx configuration copied from config/root/etc - nginx -t && \ + openresty -c /etc/nginx/nginx.conf -e stderr + -t && \ # Append software versions ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${ffmpeg_version}" && \ From f9d1e00356f97fa7f769ee5a7280e41b45e97902 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:41:57 -0400 Subject: [PATCH 009/174] fixup: remove the extra newline --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64782388..6d49e051 100644 --- a/Dockerfile +++ b/Dockerfile @@ -456,8 +456,7 @@ RUN set -x && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ # Check nginx configuration copied from config/root/etc - openresty -c /etc/nginx/nginx.conf -e stderr - -t && \ + openresty -c /etc/nginx/nginx.conf -e stderr -t && \ # Append software versions ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${ffmpeg_version}" && \ From e58c4bae99efefc6f27b2fa790fa6ad55e7f8edd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 22:13:45 -0400 Subject: [PATCH 010/174] Switch back to the `nginx-light` stage --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6d49e051..eabc4f2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -303,12 +303,14 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va set -x && \ apt-get update && \ apt-get -y --no-install-recommends install nginx-light && \ + # openresty binary should still work + ln -v -s -T ../sbin/nginx /usr/bin/openresty && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-openresty AS tubesync +FROM tubesync-nginx AS tubesync ARG S6_VERSION From 144720f6af4d9ee9f4bf289ce3fa42c750d3419c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 22:47:53 -0400 Subject: [PATCH 011/174] Switch to the `openresty` stage --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eabc4f2e..6ef178c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -310,7 +310,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-nginx AS tubesync +FROM tubesync-openresty AS tubesync ARG S6_VERSION From ca9fbd7168ef0bbf9ffc013a3f5d9c66799f8bc6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 03:25:36 -0400 Subject: [PATCH 012/174] Copy updated thumbnail next to media --- tubesync/sync/tasks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index efd03152..81dd4b5d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -555,6 +555,16 @@ def download_media_thumbnail(media_id, url): ) i = image_file = None log.info(f'Saved thumbnail for: {media} from: {url}') + # After media is downloaded, copy the updated thumbnail. + copy_thumbnail = ( + media.downloaded and + media.source.copy_thumbnails and + media.thumb_file_exists + ) + if copy_thumbnail: + log.info(f'Copying media thumbnail from: {media.thumb.path} ' + f'to: {media.thumbpath}') + copyfile(media.thumb.path, media.thumbpath) return True From ddf90122103df88aba9d7ad1635a1bd6624e68a6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 08:42:56 -0400 Subject: [PATCH 013/174] Handle `AssertionError` for timestamp --- tubesync/sync/tasks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 81dd4b5d..ed628958 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -330,9 +330,14 @@ def index_source_task(source_id): media.duration = float(video.get(fields('duration', media), None) or 0) or None media.title = str(video.get(fields('title', media), ''))[:200] timestamp = video.get(fields('timestamp', media), None) - published_dt = media.metadata_published(timestamp) - if published_dt is not None: - media.published = published_dt + if timestamp is not None: + try: + published_dt = media.metadata_published(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt try: media.save() except IntegrityError as e: From f856caa0245fdef50a2f921280f215bf9b94e67d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:21:21 -0400 Subject: [PATCH 014/174] Remove assertion --- tubesync/sync/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index eec9d7ca..3b1f66a9 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1312,7 +1312,6 @@ class Media(models.Model): return self.get_metadata_first_value(('fulltitle', 'title',), '') def ts_to_dt(self, /, timestamp): - assert timestamp is not None try: timestamp_float = float(timestamp) except Exception as e: From 0b37aa3d9f88a4c99e872eac1643a122fa143e31 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:26:14 -0400 Subject: [PATCH 015/174] Call `Media.ts_to_dt` function instead --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ed628958..79f3b375 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -332,7 +332,7 @@ def index_source_task(source_id): timestamp = video.get(fields('timestamp', media), None) if timestamp is not None: try: - published_dt = media.metadata_published(timestamp) + published_dt = media.ts_to_dt(timestamp) except AssertionError: pass else: From 39be70454509605c8ea65f7cd5d019cdc8b2d6d0 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:44:27 -0400 Subject: [PATCH 016/174] Switch to `Media.ts_to_dt` function --- tubesync/sync/tasks.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 79f3b375..39280059 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -505,9 +505,17 @@ def download_media_metadata(media_id): # Media must have a valid upload date if upload_date: media.published = timezone.make_aware(upload_date) - published = media.metadata_published() - if published: - media.published = published + timestamp = media.get_metadata_first_value( + ('release_timestamp', 'timestamp',), + arg_dict=response, + ) + try: + published_dt = media.ts_to_dt(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt # Store title in DB so it's fast to access if media.metadata_title: From c52fd14cd0c206ac750d03930bc75071d7bfe6bb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:46:48 -0400 Subject: [PATCH 017/174] Remove `Media.metadata_published` function --- tubesync/sync/models.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 3b1f66a9..05eaf854 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1321,13 +1321,6 @@ class Media(models.Model): return self.posix_epoch + timedelta(seconds=timestamp_float) return None - def metadata_published(self, timestamp=None): - if timestamp is None: - timestamp = self.get_metadata_first_value( - ('release_timestamp', 'timestamp',) - ) - return self.ts_to_dt(timestamp) - @property def slugtitle(self): replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and') From 19a2d03c5b7c436c31ff951e96d52deb50573c96 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:50:49 -0400 Subject: [PATCH 018/174] Remove an unneeded `if` --- tubesync/sync/tasks.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 39280059..5e727e6d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -330,14 +330,13 @@ def index_source_task(source_id): media.duration = float(video.get(fields('duration', media), None) or 0) or None media.title = str(video.get(fields('title', media), ''))[:200] timestamp = video.get(fields('timestamp', media), None) - if timestamp is not None: - try: - published_dt = media.ts_to_dt(timestamp) - except AssertionError: - pass - else: - if published_dt: - media.published = published_dt + try: + published_dt = media.ts_to_dt(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt try: media.save() except IntegrityError as e: From f850502ad05a12184dd186c491ad3c443f951d94 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 10:24:44 -0400 Subject: [PATCH 019/174] Narrow the exceptions --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 05eaf854..a5e3d675 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1314,7 +1314,7 @@ class Media(models.Model): def ts_to_dt(self, /, timestamp): try: timestamp_float = float(timestamp) - except Exception as e: + except (TypeError, ValueError,) as e: log.warn(f'Could not compute published from timestamp for: {self.source} / {self} with "{e}"') pass else: From cf8555aab1dc3ea308a91ad466640b84f2366695 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 14:28:20 -0400 Subject: [PATCH 020/174] Show migrations when debugging --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index ff0d4d55..baaf6e0c 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -25,6 +25,13 @@ then chmod -R 0755 /downloads fi +if [ 'True' = "${TUBESYNC_DEBUG:-False}" ] +then + s6-setuidgid app \ + /usr/bin/python3 /app/manage.py \ + showmigrations -v 3 --list +fi + # Run migrations exec s6-setuidgid app \ /usr/bin/python3 /app/manage.py migrate From f6b20c03e34431bf5d4cfbcd694b937b2f711af9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 15:36:00 -0400 Subject: [PATCH 021/174] Add `NoThumbnailException` --- tubesync/common/errors.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index 87d8aa4d..67a00eba 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -1,3 +1,6 @@ +from django.http import Http404 + + class NoMediaException(Exception): ''' Raised when a source returns no media to be indexed. Could be an invalid @@ -22,6 +25,13 @@ class NoMetadataException(Exception): pass +class NoThumbnailException(Http404): + ''' + Raised when a thumbnail was not found at the remote URL. + ''' + pass + + class DownloadFailedException(Exception): ''' Raised when a downloaded media file is expected to be present, but doesn't From 3a0c4c8fb15a4deefac2f16027f851510ea6141e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:12:25 -0400 Subject: [PATCH 022/174] Raise when `status_code` is not 200: OK --- tubesync/sync/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 917a9531..b67fedc8 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -65,6 +65,8 @@ def get_remote_image(url, force_rgb=True): '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36') } r = requests.get(url, headers=headers, stream=True, timeout=60) + if 200 != r.status_code: + r.raise_for_status() r.raw.decode_content = True i = Image.open(r.raw) if force_rgb: From add6454c336be23f869a2617fd02d94d5b20a17c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:20:48 -0400 Subject: [PATCH 023/174] Use `NoThumbnailException` from thumbnail task --- tubesync/sync/tasks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 5e727e6d..3c70c0ef 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -8,6 +8,7 @@ import os import json import math import random +import requests import time import uuid from io import BytesIO @@ -29,7 +30,8 @@ from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, - NoMetadataException, DownloadFailedException, ) + NoMetadataException, NoThumbnailException, + DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, ) from .choices import Val, TaskQueue @@ -548,7 +550,10 @@ def download_media_thumbnail(media_id, url): return width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) - i = get_remote_image(url) + try: + i = get_remote_image(url) + except requests.HTTPError as e: + raise NoThumbnailException(url) from e if (i.width > width) and (i.height > height): log.info(f'Resizing {i.width}x{i.height} thumbnail to ' f'{width}x{height}: {url}') From d6b60f41c9fe266d0d169bf168af695a4254ae75 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:52:20 -0400 Subject: [PATCH 024/174] End the task for HTTP 404 response status --- tubesync/sync/tasks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3c70c0ef..744373da 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -551,9 +551,14 @@ def download_media_thumbnail(media_id, url): width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) try: - i = get_remote_image(url) - except requests.HTTPError as e: - raise NoThumbnailException(url) from e + try: + i = get_remote_image(url) + except requests.HTTPError as re: + if 404 != re.response.status_code: + raise + raise NoThumbnailException(re.response.reason) from re + except NoThumbnailException as e: + raise InvalidTaskError(str(e.__cause__)) from e if (i.width > width) and (i.height > height): log.info(f'Resizing {i.width}x{i.height} thumbnail to ' f'{width}x{height}: {url}') From a7fdd02d47a462aac003a5de3ebd331d0d56c20f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 17:09:23 -0400 Subject: [PATCH 025/174] A standard `Exception` is fine --- tubesync/common/errors.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index 67a00eba..9ff44a48 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -1,6 +1,3 @@ -from django.http import Http404 - - class NoMediaException(Exception): ''' Raised when a source returns no media to be indexed. Could be an invalid @@ -25,7 +22,7 @@ class NoMetadataException(Exception): pass -class NoThumbnailException(Http404): +class NoThumbnailException(Exception): ''' Raised when a thumbnail was not found at the remote URL. ''' From 46d2e3c8e70243cef3134845e7857ff627034a5e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 17:14:07 -0400 Subject: [PATCH 026/174] `raise_for_status` checks `status_code` itself --- tubesync/sync/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index b67fedc8..5bc90d25 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -65,8 +65,7 @@ def get_remote_image(url, force_rgb=True): '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36') } r = requests.get(url, headers=headers, stream=True, timeout=60) - if 200 != r.status_code: - r.raise_for_status() + r.raise_for_status() r.raw.decode_content = True i = Image.open(r.raw) if force_rgb: From 71b57dd477896cfc23082d5025de6f5ee50b1fa6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 21:03:09 -0400 Subject: [PATCH 027/174] Schedule thumbnail tasks for new media --- tubesync/sync/tasks.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 744373da..304b38e6 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -354,8 +354,21 @@ def index_source_task(source_id): ) if new_media_instance: log.info(f'Indexed new media: {source} / {media}') + log.info(f'Scheduling tasks to download thumbnail for: {media.key}') + thumbnail_fmt = 'https://i.ytimg.com/vi/{}/{}default.jpg' + vn_fmt = _('Downloading {} thumbnail for: "{}": {}' + for prefix in ('hq', 'sd', 'maxres',): + thumbnail_url = thumbnail_fmt.format( + media.key, + prefix, + ) + download_media_thumbnail( + str(media.pk), + thumbnail_url, + verbose_name=vn_fmt.format(prefix, media.key, media.name), + ) log.info(f'Scheduling task to download metadata for: {media.url}') - verbose_name = _('Downloading metadata for: {}: "{}"') + verbose_name = _('Downloading metadata for: "{}": {}') download_media_metadata( str(media.pk), verbose_name=verbose_name.format(media.key, media.name), From efc59420516d480264ecda37afe155e08f12ba1a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 21:05:16 -0400 Subject: [PATCH 028/174] fixup: closing parenthesis --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 304b38e6..2e6b1433 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -356,7 +356,7 @@ def index_source_task(source_id): log.info(f'Indexed new media: {source} / {media}') log.info(f'Scheduling tasks to download thumbnail for: {media.key}') thumbnail_fmt = 'https://i.ytimg.com/vi/{}/{}default.jpg' - vn_fmt = _('Downloading {} thumbnail for: "{}": {}' + vn_fmt = _('Downloading {} thumbnail for: "{}": {}') for prefix in ('hq', 'sd', 'maxres',): thumbnail_url = thumbnail_fmt.format( media.key, From d5045d8d03232e726058b500b5a9b03b3198fb7a Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 21:13:16 -0400 Subject: [PATCH 029/174] Created fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 tubesync/sync/management/commands/fix-mariadb.py diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py new file mode 100755 index 00000000..ab9d3624 --- /dev/null +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -0,0 +1,39 @@ +from django import db +from django.utils.translation import gettext_lazy as _ +from django.core.management.base import BaseCommand, CommandError +from common.logger import log + + +class Command(BaseCommand): + + help = _('Fixes MariaDB database issues') + requires_migrations_checks = True + + def add_arguments(self, parser): + parser.add_argument( + '--uuid-columns', + action='store_true', + required=False, + help=_('Switch to the native UUID column type'), + ) + parser.add_argument( + '--delete-table', + action='store', + required=False, + help=_('Table name'), + ) + + def handle(self, *args, **options): + if 'mysql' != db.connection.vendor: + raise CommandError( + _('An invalid database vendor is configured') + f': {db.connection.vendor}' + ) + if not db.connection.mysql_is_mariadb(): + raise CommandError(_('Not conbected to a MariaDB database server.')) + + uuid_columns = options.get('uuid-columns', False) + table_name_str = options.get('delete-table', '') + + # All done + log.info('Done') From c11e78bc4704464e183e3f5fa42f021cf0ccc48e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 21:19:59 -0400 Subject: [PATCH 030/174] Remove executable bit --- tubesync/sync/management/commands/fix-mariadb.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tubesync/sync/management/commands/fix-mariadb.py diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py old mode 100755 new mode 100644 From 0f7643971ff88966cf64fe10665d6225688dfe7b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 22:44:31 -0400 Subject: [PATCH 031/174] Adjust options in fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index ab9d3624..a2e290b2 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,9 +1,16 @@ from django import db +from pprint import pp from django.utils.translation import gettext_lazy as _ from django.core.management.base import BaseCommand, CommandError from common.logger import log +def SQLTable(arg_table): + assert isinstance(arg_table, str), type(arg_table) + assert arg_table.startswith('sync_'), _('Invalid table name') + return str(arg_table) + + class Command(BaseCommand): help = _('Fixes MariaDB database issues') @@ -13,14 +20,15 @@ class Command(BaseCommand): parser.add_argument( '--uuid-columns', action='store_true', - required=False, + default=False, help=_('Switch to the native UUID column type'), ) parser.add_argument( '--delete-table', - action='store', - required=False, - help=_('Table name'), + action='append', + metavar='TABLE', + type=SQLTable, + help=_('SQL table name'), ) def handle(self, *args, **options): @@ -32,8 +40,13 @@ class Command(BaseCommand): if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) - uuid_columns = options.get('uuid-columns', False) - table_name_str = options.get('delete-table', '') + uuid_columns = options.get('uuid-columns') + table_names = options.get('delete-table', list()) + + if options['uuid-columns']: + self.stdout.write('Time to update the columns!') + + pp( table_names ) # All done log.info('Done') From a7fe0d5eeb267f48fefaf0a853c4a3af82615866 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 22:49:13 -0400 Subject: [PATCH 032/174] Combine the strings --- tubesync/sync/management/commands/fix-mariadb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index a2e290b2..bd3dd2bf 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -33,10 +33,10 @@ class Command(BaseCommand): def handle(self, *args, **options): if 'mysql' != db.connection.vendor: - raise CommandError( - _('An invalid database vendor is configured') + raise CommandError(_( + 'An invalid database vendor is configured' f': {db.connection.vendor}' - ) + )) if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) From 07f689e5d0d75edfa06242a687da6dc195df899a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 00:04:00 -0400 Subject: [PATCH 033/174] Update fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index bd3dd2bf..800b02b8 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,13 +1,18 @@ from django import db from pprint import pp -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand, CommandError from common.logger import log +def _(arg_str): + return str(gettext_lazy(arg_str)) + def SQLTable(arg_table): - assert isinstance(arg_table, str), type(arg_table) - assert arg_table.startswith('sync_'), _('Invalid table name') + if not isinstance(arg_table, str): + raise TypeError(type(arg_table)) + if not arg_table.startswith('sync_'): + raise ValueError(_('Invalid table name')) return str(arg_table) @@ -33,17 +38,16 @@ class Command(BaseCommand): def handle(self, *args, **options): if 'mysql' != db.connection.vendor: - raise CommandError(_( - 'An invalid database vendor is configured' - f': {db.connection.vendor}' - )) + raise CommandError( + _('An invalid database vendor is configured') + + f': {db.connection.vendor}' + ) if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) - uuid_columns = options.get('uuid-columns') - table_names = options.get('delete-table', list()) + table_names = options.get('delete_table', list()) - if options['uuid-columns']: + if options['uuid_columns']: self.stdout.write('Time to update the columns!') pp( table_names ) From 919ba49e28fb0c281481b12ff8f5f8d3a2350eaf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 00:21:16 -0400 Subject: [PATCH 034/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 800b02b8..a25f8da7 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -31,6 +31,7 @@ class Command(BaseCommand): parser.add_argument( '--delete-table', action='append', + default=list(), metavar='TABLE', type=SQLTable, help=_('SQL table name'), @@ -42,14 +43,20 @@ class Command(BaseCommand): _('An invalid database vendor is configured') + f': {db.connection.vendor}' ) - if not db.connection.mysql_is_mariadb(): + db_is_mariadb = ( + hasattr(db.connection, 'mysql_is_mariadb') and + db.connection.mysql_is_mariadb() + ) + if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) - table_names = options.get('delete_table', list()) + table_names = options.get('delete_table') + log.info('Start') if options['uuid_columns']: self.stdout.write('Time to update the columns!') + self.stdout.write('Tables to drop:') pp( table_names ) # All done From 960c07a84665e1f532c4db21bc06708fa937e204 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 01:20:39 -0400 Subject: [PATCH 035/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index a25f8da7..826b5533 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -11,7 +11,12 @@ def _(arg_str): def SQLTable(arg_table): if not isinstance(arg_table, str): raise TypeError(type(arg_table)) - if not arg_table.startswith('sync_'): + tables = db.connection.introspection.table_names(include_views=False) + valid_table_name = ( + arg_table.startswith('sync_') and + arg_table in tables + ) + if not valid_table_name: raise ValueError(_('Invalid table name')) return str(arg_table) @@ -43,20 +48,27 @@ class Command(BaseCommand): _('An invalid database vendor is configured') + f': {db.connection.vendor}' ) + db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and + db.connection.is_usable() and db.connection.mysql_is_mariadb() ) if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) + display_name = db.connection.display_name table_names = options.get('delete_table') log.info('Start') if options['uuid_columns']: + if 'uuid' != db.connection.data_types.get('UUIDField', ''): + raise CommandError(_( + f'The {display_name} database server does not support UUID columns.' + )) self.stdout.write('Time to update the columns!') - self.stdout.write('Tables to drop:') + self.stdout.write('Tables to delete:') pp( table_names ) # All done From e4f3428aa8c2ccaa46f1bf21251e432b4a38177d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 01:49:18 -0400 Subject: [PATCH 036/174] Allow deleting only a few tables - It must be one of the metadata related tables - The table must be present in the database --- .../sync/management/commands/fix-mariadb.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 826b5533..152111f8 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -5,16 +5,25 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db_tables = db.connection.introspection.table_names +new_tables = { + 'sync_media_metadata_format', + 'sync_media_metadata', + 'sync_metadataformat', + 'sync_metadata', +} + def _(arg_str): return str(gettext_lazy(arg_str)) def SQLTable(arg_table): - if not isinstance(arg_table, str): - raise TypeError(type(arg_table)) - tables = db.connection.introspection.table_names(include_views=False) + assert isinstance(arg_table, str), type(arg_table) + needle = arg_table + if needle.startswith('new__'): + needle = arg_table[len('new__'):] valid_table_name = ( - arg_table.startswith('sync_') and - arg_table in tables + needle in new_tables and + arg_table in db_tables(include_views=False) ) if not valid_table_name: raise ValueError(_('Invalid table name')) From 5235c53dadce7d5e72f142c41b5c3bb1cf2fee3f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 03:47:18 -0400 Subject: [PATCH 037/174] Check the current column type --- .../sync/management/commands/fix-mariadb.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 152111f8..cb6b5c1d 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -5,7 +5,9 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db_columns = db.connection.introspection.get_table_description db_tables = db.connection.introspection.table_names +db_quote_name = db.connection.ops.quote_name new_tables = { 'sync_media_metadata_format', 'sync_media_metadata', @@ -51,6 +53,20 @@ class Command(BaseCommand): help=_('SQL table name'), ) + def _get_fields(self, table_str, /): + columns = list() + with db.connection.cursor() as cursor: + columns.extend(db_columns(cursor, table_str)) + return columns + + def _using_char_for_uuid(self, table_str, /): + fields = self._get_fields(table_str) + return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] + + def _uuid_column_type(self, table_str, /): + fields = self._get_fields(table_str) + return [ f.type_code for f in fields if 'uuid' == f.name ][0] + def handle(self, *args, **options): if 'mysql' != db.connection.vendor: raise CommandError( @@ -75,7 +91,22 @@ class Command(BaseCommand): raise CommandError(_( f'The {display_name} database server does not support UUID columns.' )) - self.stdout.write('Time to update the columns!') + both_tables = ( + self._using_char_for_uuid('sync_source') and + self._using_char_for_uuid('sync_media') + ) + if not both_tables: + if 'uuid' == self._uuid_column_type('sync_source').lower(): + log.notice('The source table is already using a native UUID column.') + elif 'uuid' == self._uuid_column_type('sync_media').lower(): + log.notice('The media table is already using a native UUID column.') + else: + raise CommandError(_( + 'The database is not in an appropriate state to switch to ' + 'native UUID columns. Manual intervention is required.' + )) + else: + self.stdout.write('Time to update the columns!') self.stdout.write('Tables to delete:') pp( table_names ) From ba87e39e3d710b2820257cbb5840eddbcc862f8d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 03:58:56 -0400 Subject: [PATCH 038/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index cb6b5c1d..24eabcee 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -63,9 +63,9 @@ class Command(BaseCommand): fields = self._get_fields(table_str) return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] - def _uuid_column_type(self, table_str, /): + def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.type_code for f in fields if 'uuid' == f.name ][0] + return [ f.type_code for f in fields if column_str.lower() == f.name.lower() ][0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -96,9 +96,11 @@ class Command(BaseCommand): self._using_char_for_uuid('sync_media') ) if not both_tables: - if 'uuid' == self._uuid_column_type('sync_source').lower(): + if 'uuid' == self._column_type('sync_source', 'uuid').lower(): log.notice('The source table is already using a native UUID column.') - elif 'uuid' == self._uuid_column_type('sync_media').lower(): + elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): + log.notice('The media table is already using a native UUID column.') + elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): log.notice('The media table is already using a native UUID column.') else: raise CommandError(_( From 435ebac328d40cb7954a951f454148603016b6bb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 05:19:59 -0400 Subject: [PATCH 039/174] Generate some SQL for native UUID columns --- .../sync/management/commands/fix-mariadb.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 24eabcee..57f67804 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -14,6 +14,7 @@ new_tables = { 'sync_metadataformat', 'sync_metadata', } +sql_statements = db.connection.ops.prepare_sql_script def _(arg_str): return str(gettext_lazy(arg_str)) @@ -110,6 +111,52 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') + schema = db.connection.schema_editor() + media_table_str = db_quote_name('sync_media') + source_table_str = db_quote_name('sync_source') + fk_name_str = db_quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') + source_id_column_str = db_quote_name('source_id') + uuid_column_str = db_quote_name('uuid') + uuid_type_str = 'uuid'.upper() + remove_fk = schema.sql_delete_fk.format(dict( + table=media_table_str, + name=fk_name_str, + )) + add_fk = schema.sql_create_fk.format(dict( + table=media_table_str, + name=fk_name_str, + column=source_id_column_str, + to_table=source_table_str, + to_column=uuid_column_str, + deferrable='', + )) + statement_list = list(( + remove_fk, + schema.sql_alter_column.format(dict( + table=media_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=uuid_column_str, + )), + )), + schema.sql_alter_column.format(dict( + table=media_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=source_id_column_str, + )), + )), + schema.sql_alter_column.format(dict( + table=source_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=uuid_column_str, + )), + )), + add_fk, + )) + pp( statement_list ) + self.stdout.write('Tables to delete:') pp( table_names ) From 503bb3e5b197eeccaa3f61454472265faff32814 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 05:46:33 -0400 Subject: [PATCH 040/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 57f67804..23919d2e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -62,11 +62,11 @@ class Command(BaseCommand): def _using_char_for_uuid(self, table_str, /): fields = self._get_fields(table_str) - return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] + return 'uuid' in [ f.name for f in fields if 'varchar' == f.data_type ] def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.type_code for f in fields if column_str.lower() == f.name.lower() ][0] + return [ f.data_type for f in fields if column_str.lower() == f.name.lower() ][0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -78,7 +78,7 @@ class Command(BaseCommand): db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and - db.connection.mysql_is_mariadb() + db.connection.mysql_is_mariadb ) if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) @@ -98,11 +98,11 @@ class Command(BaseCommand): ) if not both_tables: if 'uuid' == self._column_type('sync_source', 'uuid').lower(): - log.notice('The source table is already using a native UUID column.') + log.info('The source table is already using a native UUID column.') elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): - log.notice('The media table is already using a native UUID column.') + log.info('The media table is already using a native UUID column.') elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): - log.notice('The media table is already using a native UUID column.') + log.info('The media table is already using a native UUID column.') else: raise CommandError(_( 'The database is not in an appropriate state to switch to ' From fc89178f28ea82c39033006ce6d7282c9c71a9e2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 06:24:10 -0400 Subject: [PATCH 041/174] Switch to string interpolation --- .../sync/management/commands/fix-mariadb.py | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 23919d2e..2978bcea 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -62,11 +62,18 @@ class Command(BaseCommand): def _using_char_for_uuid(self, table_str, /): fields = self._get_fields(table_str) - return 'uuid' in [ f.name for f in fields if 'varchar' == f.data_type ] + return 'uuid' in [ + f.name for f in fields if 'varchar' == f.data_type and 32 == f.display_size + ] def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.data_type for f in fields if column_str.lower() == f.name.lower() ][0] + found = [ + f'{f.data_type}({f.display_size})' for f in fields if column_str.lower() == f.name.lower() + ] + if not found: + return str() + return found[0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -92,16 +99,17 @@ class Command(BaseCommand): raise CommandError(_( f'The {display_name} database server does not support UUID columns.' )) + uuid_column_type_str = 'uuid(36)' both_tables = ( self._using_char_for_uuid('sync_source') and self._using_char_for_uuid('sync_media') ) if not both_tables: - if 'uuid' == self._column_type('sync_source', 'uuid').lower(): + if uuid_column_type_str == self._column_type('sync_source', 'uuid').lower(): log.info('The source table is already using a native UUID column.') - elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): + elif uuid_column_type_str == self._column_type('sync_media', 'uuid').lower(): log.info('The media table is already using a native UUID column.') - elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): + elif uuid_column_type_str == self._column_type('sync_media', 'source_id').lower(): log.info('The media table is already using a native UUID column.') else: raise CommandError(_( @@ -118,41 +126,41 @@ class Command(BaseCommand): source_id_column_str = db_quote_name('source_id') uuid_column_str = db_quote_name('uuid') uuid_type_str = 'uuid'.upper() - remove_fk = schema.sql_delete_fk.format(dict( + remove_fk = schema.sql_delete_fk % dict( table=media_table_str, name=fk_name_str, - )) - add_fk = schema.sql_create_fk.format(dict( + ) + add_fk = schema.sql_create_fk % dict( table=media_table_str, name=fk_name_str, column=source_id_column_str, to_table=source_table_str, to_column=uuid_column_str, deferrable='', - )) + ) statement_list = list(( remove_fk, - schema.sql_alter_column.format(dict( + schema.sql_alter_column % dict( table=media_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, - )), - )), - schema.sql_alter_column.format(dict( + ), + ), + schema.sql_alter_column % dict( table=media_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=source_id_column_str, - )), - )), - schema.sql_alter_column.format(dict( + ), + ), + schema.sql_alter_column % dict( table=source_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, - )), - )), + ), + ), add_fk, )) pp( statement_list ) From b5828801de10f240ccc8f9f7ba34ae76db97a53b Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 06:34:09 -0400 Subject: [PATCH 042/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 2978bcea..910b2276 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -138,7 +138,7 @@ class Command(BaseCommand): to_column=uuid_column_str, deferrable='', ) - statement_list = list(( + statement_list = [ f'{statement};' for statement in ( remove_fk, schema.sql_alter_column % dict( table=media_table_str, @@ -162,7 +162,7 @@ class Command(BaseCommand): ), ), add_fk, - )) + ) ] pp( statement_list ) self.stdout.write('Tables to delete:') From cc9b5446c1f15da29604630133b812b6f1b8f4fb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 07:24:54 -0400 Subject: [PATCH 043/174] Use the `quote_name` from schema --- tubesync/sync/management/commands/fix-mariadb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 910b2276..f946c22e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -120,11 +120,12 @@ class Command(BaseCommand): self.stdout.write('Time to update the columns!') schema = db.connection.schema_editor() - media_table_str = db_quote_name('sync_media') - source_table_str = db_quote_name('sync_source') - fk_name_str = db_quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') - source_id_column_str = db_quote_name('source_id') - uuid_column_str = db_quote_name('uuid') + quote_name = schema.quote_name + media_table_str = quote_name('sync_media') + source_table_str = quote_name('sync_source') + fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') + source_id_column_str = quote_name('source_id') + uuid_column_str = quote_name('uuid') uuid_type_str = 'uuid'.upper() remove_fk = schema.sql_delete_fk % dict( table=media_table_str, From 2a98707254c83908931da73b1be4443388cc701e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 07:46:36 -0400 Subject: [PATCH 044/174] Collect SQL with schema editor --- .../sync/management/commands/fix-mariadb.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index f946c22e..9c7ceb87 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -119,7 +119,7 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') - schema = db.connection.schema_editor() + schema = db.connection.schema_editor(collect_sql=True) quote_name = schema.quote_name media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') @@ -139,8 +139,9 @@ class Command(BaseCommand): to_column=uuid_column_str, deferrable='', ) - statement_list = [ f'{statement};' for statement in ( - remove_fk, + + schema.execute(remove_fk, None) + schema.execute( schema.sql_alter_column % dict( table=media_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -148,6 +149,9 @@ class Command(BaseCommand): column=uuid_column_str, ), ), + None, + ) + schema.execute( schema.sql_alter_column % dict( table=media_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -155,6 +159,9 @@ class Command(BaseCommand): column=source_id_column_str, ), ), + None, + ) + schema.execute( schema.sql_alter_column % dict( table=source_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -162,9 +169,11 @@ class Command(BaseCommand): column=uuid_column_str, ), ), - add_fk, - ) ] - pp( statement_list ) + None, + ) + schema.execute(add_fk, None) + + pp( schema.collected_sql ) self.stdout.write('Tables to delete:') pp( table_names ) From 36681aadc0ac66943147bb1e17d38a8b2a54b3ad Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 09:35:04 -0400 Subject: [PATCH 045/174] Add `check_migration_status` function --- .../sync/management/commands/fix-mariadb.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 9c7ceb87..3bbce4c5 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,6 +1,8 @@ from django import db +from io import BytesIO, TextIOWrapper from pprint import pp from django.utils.translation import gettext_lazy +from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError from common.logger import log @@ -32,6 +34,32 @@ def SQLTable(arg_table): raise ValueError(_('Invalid table name')) return str(arg_table) +def _mk_wrapper(): + return TextIOWrapper( + BytesIO(), + line_buffering=True, + write_through=True, + ) + +def check_migration_status(migration_str, /): + needle = 'No planned migration operations.' + wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() + call_command( + 'migrate', '-v', '3', '--plan', 'sync', + migration_str, + stderr=wrap_stderr, + stdout=wrap_stdout, + ) + wrap_stderr.seek(0, 0) + stderr_lines = wrap_stderr.readlines() + wrap_stdout.seek(0, 0) + stdout_lines = wrap_stdout.readlines() + return ( + bool([ line.decode() for line in stdout_lines if needle.encode() in line ]), + stderr_lines, + stdout_lines, + ) + class Command(BaseCommand): From e6e4c3300af8cf8b504057c7e6de1c07581fea63 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 09:41:21 -0400 Subject: [PATCH 046/174] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 3bbce4c5..39735d6e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -203,8 +203,16 @@ class Command(BaseCommand): pp( schema.collected_sql ) - self.stdout.write('Tables to delete:') - pp( table_names ) + if table_names: + pp( check_migration_status( + '0030_alter_source_source_vcodec', + )) + pp( check_migration_status( + '0031_squashed_metadata_metadataformat', + )) + + self.stdout.write('Tables to delete:') + pp( table_names ) # All done log.info('Done') From 5e2200382f35b30646374fc2539f3d11e4e53977 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 10:10:34 -0400 Subject: [PATCH 047/174] Check for completed data migration --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 39735d6e..387e913a 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -55,7 +55,7 @@ def check_migration_status(migration_str, /): wrap_stdout.seek(0, 0) stdout_lines = wrap_stdout.readlines() return ( - bool([ line.decode() for line in stdout_lines if needle.encode() in line ]), + bool([ line.decode() for line in stdout_lines if needle in line ]), stderr_lines, stdout_lines, ) @@ -64,7 +64,7 @@ def check_migration_status(migration_str, /): class Command(BaseCommand): help = _('Fixes MariaDB database issues') - requires_migrations_checks = True + requires_migrations_checks = False def add_arguments(self, parser): parser.add_argument( @@ -204,12 +204,18 @@ class Command(BaseCommand): pp( schema.collected_sql ) if table_names: + pp( check_migration_status( '0029', ) ) pp( check_migration_status( '0030_alter_source_source_vcodec', )) pp( check_migration_status( '0031_squashed_metadata_metadataformat', )) + pp( check_migration_status( '0032_metadata_transfer', )) + if check_migration_status('0032_metadata_transfer')[0]: + raise CommandError(_( + 'Deleting tables that are in use is not safe!' + )) self.stdout.write('Tables to delete:') pp( table_names ) From 5e8cc639f76b75f2f8cfb94efc5c37beda57d94c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 11:03:26 -0400 Subject: [PATCH 048/174] Generate SQL for deleting tables --- .../sync/management/commands/fix-mariadb.py | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 387e913a..1e5a9ff1 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -41,8 +41,9 @@ def _mk_wrapper(): write_through=True, ) -def check_migration_status(migration_str, /): - needle = 'No planned migration operations.' +def check_migration_status(migration_str, /, *, needle=None): + if needle is None: + needle = 'No planned migration operations.' wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() call_command( 'migrate', '-v', '3', '--plan', 'sync', @@ -55,7 +56,7 @@ def check_migration_status(migration_str, /): wrap_stdout.seek(0, 0) stdout_lines = wrap_stdout.readlines() return ( - bool([ line.decode() for line in stdout_lines if needle in line ]), + bool([ line for line in stdout_lines if needle in line ]), stderr_lines, stdout_lines, ) @@ -120,6 +121,8 @@ class Command(BaseCommand): display_name = db.connection.display_name table_names = options.get('delete_table') + schema = db.connection.schema_editor(collect_sql=True) + quote_name = schema.quote_name log.info('Start') if options['uuid_columns']: @@ -147,8 +150,6 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') - schema = db.connection.schema_editor(collect_sql=True) - quote_name = schema.quote_name media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') @@ -204,21 +205,34 @@ class Command(BaseCommand): pp( schema.collected_sql ) if table_names: - pp( check_migration_status( '0029', ) ) - pp( check_migration_status( + # Check that the migration is at an appropriate step + at_30, err_30, out_30 = check_migration_status( '0030_alter_source_source_vcodec' ) + at_31, err_31, out_31 = check_migration_status( '0031_metadata_metadataformat' ) + at_31s, err_31s, out_31s = check_migration_status( '0031_squashed_metadata_metadataformat' ) + after_31, err_31a, out_31a = check_migration_status( '0030_alter_source_source_vcodec', - )) - pp( check_migration_status( - '0031_squashed_metadata_metadataformat', - )) - pp( check_migration_status( '0032_metadata_transfer', )) - if check_migration_status('0032_metadata_transfer')[0]: + needle='Undo Rename table for metadata to sync_media_metadata', + ) + + should_delete = ( + not (at_31s or after_31) and + (at_30 or at_31) + ) + if not should_delete: raise CommandError(_( - 'Deleting tables that are in use is not safe!' + 'Deleting metadata tables that are in use is not safe!' )) self.stdout.write('Tables to delete:') pp( table_names ) + for table in table_names: + schema.execute( + schema.sql_delete_table % dict( + table=quote_name(table), + ), + None, + ) + pp( schema.collected_sql ) # All done log.info('Done') From da06a1ffa1f9468a717bb719e173ffff09fe025d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:00:28 -0400 Subject: [PATCH 049/174] Add `--dry-run` and apply SQL when it is not used --- .../sync/management/commands/fix-mariadb.py | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 1e5a9ff1..7f19ff97 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -7,7 +7,6 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log -db_columns = db.connection.introspection.get_table_description db_tables = db.connection.introspection.table_names db_quote_name = db.connection.ops.quote_name new_tables = { @@ -45,12 +44,15 @@ def check_migration_status(migration_str, /, *, needle=None): if needle is None: needle = 'No planned migration operations.' wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() - call_command( - 'migrate', '-v', '3', '--plan', 'sync', - migration_str, - stderr=wrap_stderr, - stdout=wrap_stdout, - ) + try: + call_command( + 'migrate', '-v', '3', '--plan', 'sync', + migration_str, + stderr=wrap_stderr, + stdout=wrap_stdout, + ) + except db.migrations.exceptions.NodeNotFoundError: + return (False, None, None,) wrap_stderr.seek(0, 0) stderr_lines = wrap_stderr.readlines() wrap_stdout.seek(0, 0) @@ -61,13 +63,27 @@ def check_migration_status(migration_str, /, *, needle=None): stdout_lines, ) +def db_columns(table_str, /): + columns = list() + db_gtd = db.connection.introspection.get_table_description + with db.connection.cursor() as cursor: + columns.extend(db_gtd(cursor, table_str)) + return columns + class Command(BaseCommand): help = _('Fixes MariaDB database issues') + output_transaction = True requires_migrations_checks = False def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + default=False, + help=_('Only show the SQL; do not apply it to the database'), + ) parser.add_argument( '--uuid-columns', action='store_true', @@ -80,25 +96,21 @@ class Command(BaseCommand): default=list(), metavar='TABLE', type=SQLTable, - help=_('SQL table name'), + help=_('SQL table name to be deleted'), ) - def _get_fields(self, table_str, /): - columns = list() - with db.connection.cursor() as cursor: - columns.extend(db_columns(cursor, table_str)) - return columns - - def _using_char_for_uuid(self, table_str, /): - fields = self._get_fields(table_str) - return 'uuid' in [ - f.name for f in fields if 'varchar' == f.data_type and 32 == f.display_size + def _using_char(self, table_str, column_str='uuid', /): + cols = db_columns(table_str) + char_sizes = { 32, 36, } + char_types = { 'char', 'varchar', } + return column_str in [ + c.name for c in cols if c.data_type in char_types and c.display_size in char_sizes ] def _column_type(self, table_str, column_str='uuid', /): - fields = self._get_fields(table_str) + cols = db_columns(table_str) found = [ - f'{f.data_type}({f.display_size})' for f in fields if column_str.lower() == f.name.lower() + f'{c.data_type}({c.display_size})' for c in cols if column_str.lower() == c.name.lower() ] if not found: return str() @@ -125,6 +137,8 @@ class Command(BaseCommand): quote_name = schema.quote_name log.info('Start') + + if options['uuid_columns']: if 'uuid' != db.connection.data_types.get('UUIDField', ''): raise CommandError(_( @@ -132,8 +146,8 @@ class Command(BaseCommand): )) uuid_column_type_str = 'uuid(36)' both_tables = ( - self._using_char_for_uuid('sync_source') and - self._using_char_for_uuid('sync_media') + self._using_char('sync_source', 'uuid') and + self._using_char('sync_media', 'uuid') ) if not both_tables: if uuid_column_type_str == self._column_type('sync_source', 'uuid').lower(): @@ -148,8 +162,6 @@ class Command(BaseCommand): 'native UUID columns. Manual intervention is required.' )) else: - self.stdout.write('Time to update the columns!') - media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') @@ -202,7 +214,6 @@ class Command(BaseCommand): ) schema.execute(add_fk, None) - pp( schema.collected_sql ) if table_names: # Check that the migration is at an appropriate step @@ -223,8 +234,6 @@ class Command(BaseCommand): 'Deleting metadata tables that are in use is not safe!' )) - self.stdout.write('Tables to delete:') - pp( table_names ) for table in table_names: schema.execute( schema.sql_delete_table % dict( @@ -232,7 +241,15 @@ class Command(BaseCommand): ), None, ) - pp( schema.collected_sql ) + + if not options['dry_run']: + with db.connection.schema_editor(collect_sql=False) as schema_editor: + for sql in schema.collected_sql: + schema_editor.execute(sql, None) + else: + for sql in schema.collected_sql: + self.stdout.write(sql) + # All done log.info('Done') From a59fb8bc26355839d70402398a405a48b13c68ce Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:13:23 -0400 Subject: [PATCH 050/174] The SQL output needs to be returned --- tubesync/sync/management/commands/fix-mariadb.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 7f19ff97..5fb9d4f0 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -242,13 +242,12 @@ class Command(BaseCommand): None, ) - if not options['dry_run']: + if options['dry_run']: + return '\n'.join(schema.collected_sql) + else: with db.connection.schema_editor(collect_sql=False) as schema_editor: for sql in schema.collected_sql: schema_editor.execute(sql, None) - else: - for sql in schema.collected_sql: - self.stdout.write(sql) # All done From 621a5787879e633bd74b851e4012700a41eb3f46 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:25:29 -0400 Subject: [PATCH 051/174] Log the `Done` before returning --- tubesync/sync/management/commands/fix-mariadb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 5fb9d4f0..4c515039 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -243,6 +243,7 @@ class Command(BaseCommand): ) if options['dry_run']: + log.info('Done') return '\n'.join(schema.collected_sql) else: with db.connection.schema_editor(collect_sql=False) as schema_editor: From 305d27a7af4c2f9c12cc66869aac365e02a119bf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:35:51 -0400 Subject: [PATCH 052/174] Try using `foreign_key_checks` --- tubesync/sync/management/commands/fix-mariadb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 4c515039..d9b608ec 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -181,7 +181,8 @@ class Command(BaseCommand): deferrable='', ) - schema.execute(remove_fk, None) + schema.execute('SET foreign_key_checks=0', None) + #schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( table=media_table_str, @@ -212,7 +213,8 @@ class Command(BaseCommand): ), None, ) - schema.execute(add_fk, None) + #schema.execute(add_fk, None) + schema.execute('SET foreign_key_checks=1', None) if table_names: From a8ee30192cf117e6fe55e828fdfe688407aef5c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 14:06:34 -0400 Subject: [PATCH 053/174] Remove `pprint` as it is now unused --- tubesync/sync/management/commands/fix-mariadb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index d9b608ec..c7d176e6 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,6 +1,5 @@ from django import db from io import BytesIO, TextIOWrapper -from pprint import pp from django.utils.translation import gettext_lazy from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError From c3b5a25b9d51e1b095dbea301fd14f38f5c1a1ba Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 21:36:43 -0400 Subject: [PATCH 054/174] Alter `sync_source` first --- .../sync/management/commands/fix-mariadb.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index c7d176e6..45098b3e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -182,6 +182,16 @@ class Command(BaseCommand): schema.execute('SET foreign_key_checks=0', None) #schema.execute(remove_fk, None) + schema.execute( + schema.sql_alter_column % dict( + table=source_table_str, + changes=schema.sql_alter_column_not_null % dict( + type=uuid_type_str, + column=uuid_column_str, + ), + ), + None, + ) schema.execute( schema.sql_alter_column % dict( table=media_table_str, @@ -202,16 +212,6 @@ class Command(BaseCommand): ), None, ) - schema.execute( - schema.sql_alter_column % dict( - table=source_table_str, - changes=schema.sql_alter_column_not_null % dict( - type=uuid_type_str, - column=uuid_column_str, - ), - ), - None, - ) #schema.execute(add_fk, None) schema.execute('SET foreign_key_checks=1', None) From 0fb7520a2b022bad451869def2d80188570bf0c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 22:25:37 -0400 Subject: [PATCH 055/174] Go back to removing then adding the foreign key --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 45098b3e..c3f2287a 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -180,11 +180,9 @@ class Command(BaseCommand): deferrable='', ) - schema.execute('SET foreign_key_checks=0', None) - #schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( - table=source_table_str, + table=media_table_str, changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, @@ -192,9 +190,10 @@ class Command(BaseCommand): ), None, ) + schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( - table=media_table_str, + table=source_table_str, changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, @@ -212,8 +211,7 @@ class Command(BaseCommand): ), None, ) - #schema.execute(add_fk, None) - schema.execute('SET foreign_key_checks=1', None) + schema.execute(add_fk, None) if table_names: From c593933bfa38a2e21fee35424113229c10d49b95 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 5 May 2025 05:31:17 -0400 Subject: [PATCH 056/174] Add more details about saving database entries --- docs/other-database-backends.md | 39 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index 4f90d3ab..dbccb579 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -18,22 +18,49 @@ reset your database. If you are comfortable with Django you can export and re-im existing database data with: ```bash -$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json +$ docker exec -t tubesync \ + python3 /app/manage.py \ + dumpdata --format jsonl \ + --exclude background_task \ + --output /downloads/tubesync-database-backup.jsonl.xz ``` -Then change you database backend over, then use +Writing the compressed backup file to your `/downloads/` makes sense, as long as that directory is still available after destroying the current container. +If you have a configuration where that file will be deleted, choose a different place to store the output (perhaps `/config/`, if it has sufficient storage available) and place the file there instead. + +You can also copy the file from the container to the local filesystem (`/tmp/` in this example) with: ```bash -$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json +$ docker cp \ + tubesync:/downloads/tubesync-database-backup.jsonl.xz \ + /tmp/ +``` + +If you use `-` as the destination, then `docker cp` provides a `tar` archive. + +After you have changed your database backend over, then use: + +```bash +$ docker exec -t tubesync \ + python3 /app/manage.py \ + loaddata /downloads/tubesync-database-backup.jsonl.xz +``` + +Or, if you only have the copy in `/tmp/`, then you would use: +```bash +$ xzcat /tmp/tubesync-database-backup.jsonl.xz | \ + docker exec -i tubesync \ + python3 /app/manage.py \ + loaddata --format=jsonl - ``` As detailed in the Django documentation: -https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata +https://docs.djangoproject.com/en/5.1/ref/django-admin/#dumpdata and: -https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata +https://docs.djangoproject.com/en/5.1/ref/django-admin/#loaddata Further instructions are beyond the scope of TubeSync documenation and you should refer to Django documentation for more details. @@ -94,7 +121,7 @@ the DB for the performance benefits, a configuration like this would be enough: ``` tubesync-db: - image: postgres:15.2 + image: postgres:17 container_name: tubesync-db restart: unless-stopped volumes: From e90721722b3f3b3c8936893094f11154358dec5a Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 5 May 2025 06:06:38 -0400 Subject: [PATCH 057/174] Let `missing_pot` formats download after testing --- tubesync/sync/youtube.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 55046c81..4bf4f392 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -356,6 +356,7 @@ def download_media( 'sleep_interval': 10, 'max_sleep_interval': min(20*60, max(60, settings.DOWNLOAD_MEDIA_DELAY)), 'sleep_interval_requests': 1 + (2 * settings.BACKGROUND_TASK_ASYNC_THREADS), + 'extractor_args': opts.get('extractor_args', dict()), 'paths': opts.get('paths', dict()), 'postprocessor_args': opts.get('postprocessor_args', dict()), 'postprocessor_hooks': opts.get('postprocessor_hooks', list()), @@ -379,6 +380,18 @@ def download_media( 'temp': str(temp_dir_path), }) + # Allow download of formats that tested good with 'missing_pot' + youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict()) + formats_list = youtube_ea_dict.get('formats', list()) + if 'missing_pot' not in formats_list: + formats_list += ('missing_pot',) + youtube_ea_dict.update({ + 'formats': formats_list, + }) + ytopts['extractor_args'].update({ + 'youtube': youtube_ea_dict, + }) + postprocessor_hook_func = postprocessor_hook.get('function', None) if postprocessor_hook_func: ytopts['postprocessor_hooks'].append(postprocessor_hook_func) From 024ced3696f520185a032175b23f14dfa22f49a1 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 11:44:39 -0400 Subject: [PATCH 058/174] Remux video for combined format download --- tubesync/sync/youtube.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 55046c81..14c7f06b 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -336,12 +336,15 @@ def download_media( ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds + else: + pp_opts.remuxvideo = extension ytopts = { 'format': media_format, 'final_ext': extension, 'merge_output_format': extension, 'outtmpl': os.path.basename(output_file), + 'remuxvideo': pp_opts.remuxvideo, 'quiet': False if settings.DEBUG else True, 'verbose': True if settings.DEBUG else False, 'noprogress': None if settings.DEBUG else True, From 27ceecadc85616a20b723cbf7e8f209ab78f13ff Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 11:49:18 -0400 Subject: [PATCH 059/174] Check `media_format` for `+` --- tubesync/sync/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 14c7f06b..46840338 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -336,7 +336,7 @@ def download_media( ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds - else: + elif '+' not in media_format: pp_opts.remuxvideo = extension ytopts = { From a2722ee18866b198e5ee3d9569d2fc201835dca4 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 18:01:45 -0400 Subject: [PATCH 060/174] Use `POSTGRES_DB` instead of `init.sql` --- docs/other-database-backends.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index dbccb579..d623ddd5 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -125,9 +125,9 @@ the DB for the performance benefits, a configuration like this would be enough: container_name: tubesync-db restart: unless-stopped volumes: - - //init.sql:/docker-entrypoint-initdb.d/init.sql - //tubesync-db:/var/lib/postgresql/data environment: + - POSTGRES_DB=tubesync - POSTGRES_USER=postgres - POSTGRES_PASSWORD=testpassword @@ -145,15 +145,3 @@ the DB for the performance benefits, a configuration like this would be enough: depends_on: - tubesync-db ``` - -Note that an `init.sql` file is needed to initialize the `tubesync` -database before it can be written to. This file should contain: - -``` -CREATE DATABASE tubesync; -``` - - -Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it -to be executed on first startup of the container. See the `tubesync-db` -volume mapping above for how to do this. From ed90a238308ff0648dbac1f9354657e12d1835b7 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 19:34:28 -0400 Subject: [PATCH 061/174] Stop services before database steps --- docs/other-database-backends.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index dbccb579..3d35ece2 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -18,6 +18,14 @@ reset your database. If you are comfortable with Django you can export and re-im existing database data with: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Backup the database into a compressed file $ docker exec -t tubesync \ python3 /app/manage.py \ dumpdata --format jsonl \ @@ -41,6 +49,14 @@ If you use `-` as the destination, then `docker cp` provides a `tar` archive. After you have changed your database backend over, then use: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Load fixture file into the database $ docker exec -t tubesync \ python3 /app/manage.py \ loaddata /downloads/tubesync-database-backup.jsonl.xz @@ -48,6 +64,14 @@ $ docker exec -t tubesync \ Or, if you only have the copy in `/tmp/`, then you would use: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Load fixture data from standard input into the database $ xzcat /tmp/tubesync-database-backup.jsonl.xz | \ docker exec -i tubesync \ python3 /app/manage.py \ From dab458c9f766c0583a1452836b6bf72b8f1c2f3b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 15:24:23 -0400 Subject: [PATCH 062/174] Add `music.youtube.com` to the YouTube domains --- tubesync/sync/choices.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py index 25dd762a..6412ad14 100644 --- a/tubesync/sync/choices.py +++ b/tubesync/sync/choices.py @@ -8,6 +8,7 @@ DOMAINS = dict({ 'youtube': frozenset({ 'youtube.com', 'm.youtube.com', + 'music.youtube.com', 'www.youtube.com', }), }) From e6a6552c3c3edcb0b8704ef73758ab3bdcfb2752 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 20:56:51 -0400 Subject: [PATCH 063/174] Add a start link to scheduled tasks --- tubesync/sync/templates/sync/tasks.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 9cb9dfe1..86de0441 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -67,9 +67,10 @@
{% for task in scheduled %} - {{ task }}
+ {{ task }}
{% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }}{% endif %} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + {% endif %} {% empty %} There are no scheduled tasks on this page. From 01705f5f68e18e4dbad850bb4ecf696e2a4d5418 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 21:23:42 -0400 Subject: [PATCH 064/174] Add task scheduling URLs --- tubesync/sync/urls.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 9cec74ee..72ed241a 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -122,6 +122,18 @@ urlpatterns = [ name='tasks', ), + path( + 'task//schedule/now', + TasksView.as_view(), + name='run-task', + ), + + path( + 'task//schedule/', + TasksView.as_view(), + name='schedule-task', + ), + path( 'tasks-completed', CompletedTasksView.as_view(), From dc38c0a37a78d8b3bacbb36dca3f0d545827c15d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 21:25:04 -0400 Subject: [PATCH 065/174] Match the correct URL --- tubesync/sync/templates/sync/tasks.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 86de0441..fa857b1a 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -69,7 +69,7 @@ {{ task }}
{% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} {% endif %} {% empty %} From 3b17f693464820cf75ca235a17d2b6247c839da2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 07:55:06 -0400 Subject: [PATCH 066/174] Add `TaskScheduleView` --- tubesync/sync/views.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 73fb3ddd..5c3e2cff 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -24,7 +24,7 @@ from common.utils import append_uri_params from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, - SkipMediaForm, EnableMediaForm, ResetTasksForm, + SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, ConfirmDeleteMediaServerForm) from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, @@ -1004,6 +1004,41 @@ class ResetTasks(FormView): return append_uri_params(url, {'message': 'reset'}) +class TaskScheduleView(FormView, SingleObjectMixin): + ''' + Confirm that the task should be re-scheduled. + ''' + + template_name = 'sync/task-schedule.html' + form_class = ScheduleTaskForm + model = Task + + def __init__(self, *args, **kwargs): + self.object = None + self.schedule = 0 + super().__init__(*args, **kwargs) + + def dispatch(self, request, *args, **kwargs): + self.object = self.get_object() + return super().dispatch(request, *args, **kwargs) + + def get_success_url(self): + return append_uri_params( + reverse_lazy('sync:tasks'), + dict( + message='scheduled', + pk=str(self.object.pk), + ), + ) + + def form_valid(self, form): + max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) + self.object.attempts = max_attempts // 2 + self.object.run_at = timezone.now() + timezone.timedelta(seconds=self.schedule) + self.object.save() + return super().form_valid(form) + + class MediaServersView(ListView): ''' List of media servers which have been added. From 317f2939919a45e9be6766d1fb64477045e6d50f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 07:58:56 -0400 Subject: [PATCH 067/174] Add `ScheduleTaskForm` --- tubesync/sync/forms.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 3d795a5f..cf73f8b4 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -44,10 +44,16 @@ class ResetTasksForm(forms.Form): pass +class ScheduleTaskForm(forms.Form): + + pass + + class ConfirmDeleteMediaServerForm(forms.Form): pass + _media_server_type_label = 'Jellyfin' class JellyfinMediaServerForm(forms.Form): From 8da507c9350bb17cc45ae1dcc165538c38430a2a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:15:51 -0400 Subject: [PATCH 068/174] Create task-schedule.html --- .../sync/templates/sync/task-schedule.html | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tubesync/sync/templates/sync/task-schedule.html diff --git a/tubesync/sync/templates/sync/task-schedule.html b/tubesync/sync/templates/sync/task-schedule.html new file mode 100644 index 00000000..8f387b4a --- /dev/null +++ b/tubesync/sync/templates/sync/task-schedule.html @@ -0,0 +1,34 @@ +{% extends 'base.html' %} + +{% block headtitle %}Schedule task{% endblock %} + +{% block content %} +
+
+

Schedule task

+

+ If you don't want to wait for the existing schedule to be triggered, + you can use this to change when the task will be scheduled to run. + It is not guaranteed to run at any exact time, because when a task + requests to run and when a slot to execute it, in the appropriate + queue and with the priority level assigned, is dependent on how long + other tasks are taking to complete the assigned work. +

+

+ This will change the time that the task is requesting to be run + to the current time, plus some number of seconds. +

+
+
+
+
+ {% csrf_token %} + {% include 'simpleform.html' with form=form %} +
+
+ +
+
+
+
+{% endblock %} From 30b77d182b1f3d0a150bcfceae4b0973dfa21a09 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:19:20 -0400 Subject: [PATCH 069/174] Use `TaskScheduleView` --- tubesync/sync/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 72ed241a..232c21dc 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -3,7 +3,7 @@ from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceVie SourceView, UpdateSourceView, DeleteSourceView, MediaView, MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView, MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks, - MediaServersView, AddMediaServerView, MediaServerView, + TaskScheduleView, MediaServersView, AddMediaServerView, MediaServerView, DeleteMediaServerView, UpdateMediaServerView) @@ -124,7 +124,7 @@ urlpatterns = [ path( 'task//schedule/now', - TasksView.as_view(), + TaskScheduleView.as_view(), name='run-task', ), From d8eca035acb89553495a99a2621d02d199412e1f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:22:34 -0400 Subject: [PATCH 070/174] Update urls.py --- tubesync/sync/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 232c21dc..2dffea5d 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -130,7 +130,7 @@ urlpatterns = [ path( 'task//schedule/', - TasksView.as_view(), + TaskScheduleView.as_view(), name='schedule-task', ), From 329c13625c8d3d960c396911b4d1cf7e2c730f31 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 09:28:49 -0400 Subject: [PATCH 071/174] Add a missing error message --- tubesync/sync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 73fb3ddd..bdedb3fe 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -168,6 +168,7 @@ class ValidateSourceView(FormView): template_name = 'sync/source-validate.html' form_class = ValidateSourceForm errors = { + 'invalid_source': _('Invalid type for the source.'), 'invalid_url': _('Invalid URL, the URL must for a "{item}" must be in ' 'the format of "{example}". The error was: {error}.'), } From 4f685527132c3c8d53ed5c3a56bc0c90e8c1a1ef Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 09:50:01 -0400 Subject: [PATCH 072/174] Provide `when` for the form and the template --- tubesync/sync/views.py | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 5c3e2cff..218d259e 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -20,6 +20,7 @@ from django.utils.text import slugify from django.utils._os import safe_join from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from common.timestamp import timestamp_to_datetime from common.utils import append_uri_params from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer @@ -1012,16 +1013,42 @@ class TaskScheduleView(FormView, SingleObjectMixin): template_name = 'sync/task-schedule.html' form_class = ScheduleTaskForm model = Task + errors = dict( + invalid_when=_('The type ({}) was incorrect.'), + ) def __init__(self, *args, **kwargs): self.object = None - self.schedule = 0 + self.timestamp = None + self.when = None super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): self.object = self.get_object() + self.timestamp = kwargs.get('timestamp', '') + try: + self.timestamp = int(self.timestamp, 10) + except (TypeError, ValueError): + self.timestamp = None + else: + try: + self.when = timestamp_to_datetime(self.timestamp) + except AssertionError: + self.when = None + if self.when is None: + self.when = timezone.now() return super().dispatch(request, *args, **kwargs) + def get_initial(self): + initial = super().get_initial() + initial['when'] = self.when + return initial + + def get_context_data(self, *args, **kwargs): + data = super().get_context_data(*args, **kwargs) + data['when'] = self.when + return data + def get_success_url(self): return append_uri_params( reverse_lazy('sync:tasks'), @@ -1033,9 +1060,26 @@ class TaskScheduleView(FormView, SingleObjectMixin): def form_valid(self, form): max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) + now = timezone.now() + when = form.cleaned_data.get('when') + + if not isinstance(when, now.__class__): + form.add_error( + 'when', + ValidationError( + errors['invalid_when'].format( + type(when), + ), + ), + ) + + if form.errors: + return super().form_invalid(form) + self.object.attempts = max_attempts // 2 - self.object.run_at = timezone.now() + timezone.timedelta(seconds=self.schedule) + self.object.run_at = when self.object.save() + return super().form_valid(form) From 0e29a5bdbe84974f9dc136af8fd9028750d3b428 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:06:04 -0400 Subject: [PATCH 073/174] Add `when` to the form --- tubesync/sync/forms.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index cf73f8b4..c0bfd13f 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -46,7 +46,10 @@ class ResetTasksForm(forms.Form): class ScheduleTaskForm(forms.Form): - pass + when = forms.DateTimeField( + label=_('When the task should run'), + required=True, + ) class ConfirmDeleteMediaServerForm(forms.Form): From 90253ecab63f3660602ccd79ab4396500adac92a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:27:22 -0400 Subject: [PATCH 074/174] Display both links in the same item --- tubesync/sync/templates/sync/tasks.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index fa857b1a..1bf7b2c4 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -66,12 +66,17 @@

{% for task in scheduled %} - - {{ task }}
- {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} - {% endif %} - +
+ + {{ task }}
+ {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + + + + {% endif %} + +
{% empty %} There are no scheduled tasks on this page. {% endfor %} From dc3e320c11dc3a68c80eae9d4827ce8fd61ba177 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:36:03 -0400 Subject: [PATCH 075/174] Allow running tasks with errors also --- tubesync/sync/templates/sync/tasks.html | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 1bf7b2c4..33f6c40b 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -43,11 +43,16 @@

{% for task in errors %} - - {{ task }}, attempted {{ task.attempts }} time{{ task.attempts|pluralize }}
- Error: "{{ task.error_message }}"
+
+ + {{ task }}, attempted {{ task.attempts }} time{{ task.attempts|pluralize }}
+ Error: "{{ task.error_message }}"
+ Task will be retried at {{ task.run_at|date:'Y-m-d H:i:s' }} - + + + +
{% empty %} There are no tasks with errors on this page. {% endfor %} From f5fcfd2e1efd8e6410bdadf18d4f8ffd80edc1f4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 15:54:13 -0400 Subject: [PATCH 076/174] Log `OSError` when checking for existing media files --- tubesync/sync/signals.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 32b0b5f6..06c0e2fa 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -272,8 +272,15 @@ def media_post_save(sender, instance, created, **kwargs): thumbnail_url, verbose_name=verbose_name.format(instance.name), ) + media_file_exists = False + try: + media_file_exists |= instance.media_file_exists + media_file_exists |= instance.filepath.exists() + except OSError as e: + log.exception(e) + pass # If the media has not yet been downloaded schedule it to be downloaded - if not (instance.media_file_exists or instance.filepath.exists() or existing_media_download_task): + if not (media_file_exists or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. if instance.can_download and instance.downloaded: skip_changed = True != instance.skip From 266218f8163ed733400801c7b041292d8ad2c008 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:48:43 -0400 Subject: [PATCH 077/174] Rename tubesync/sync/models.py to tubesync/sync/models/__init__.py --- tubesync/sync/{models.py => models/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/{models.py => models/__init__.py} (100%) diff --git a/tubesync/sync/models.py b/tubesync/sync/models/__init__.py similarity index 100% rename from tubesync/sync/models.py rename to tubesync/sync/models/__init__.py From 40b7ce1d2eaf706061a3d535a5ff17b4ee427804 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:51:22 -0400 Subject: [PATCH 078/174] Create media_server.py --- tubesync/sync/models/media_server.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/media_server.py diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/media_server.py @@ -0,0 +1 @@ + From a8b811abcdb30fae8c8c82037e150768a2cb067f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:54:08 -0400 Subject: [PATCH 079/174] Create metadata_format.py --- tubesync/sync/models/metadata_format.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/metadata_format.py diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/metadata_format.py @@ -0,0 +1 @@ + From e764e43e6d20480b8dd17032d9f4376751c36a60 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:55:45 -0400 Subject: [PATCH 080/174] Create metadata.py --- tubesync/sync/models/metadata.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/metadata.py diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/metadata.py @@ -0,0 +1 @@ + From e33cf5220930a7917a546b2ef4bf45d56f95da6a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:56:11 -0400 Subject: [PATCH 081/174] Create source.py --- tubesync/sync/models/source.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/source.py diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/source.py @@ -0,0 +1 @@ + From 6717f1a58b66d41a8d10385c081b1fb4eac0d380 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:57:35 -0400 Subject: [PATCH 082/174] Create media.py --- tubesync/sync/models/media.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/media.py diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/media.py @@ -0,0 +1 @@ + From 182836aa990e2839bd236cbfd849a813068397c7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 21:33:55 -0400 Subject: [PATCH 083/174] Update media_server.py --- tubesync/sync/models/media_server.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index 8b137891..5e5e8c30 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1 +1,89 @@ +import json +from django import db +from django.utils.translation import gettext_lazy as _ +from ..choices import Val, MediaServerType + +class MediaServer(db.models.Model): + ''' + A remote media server, such as a Plex server. + ''' + + ICONS = { + Val(MediaServerType.JELLYFIN): '', + Val(MediaServerType.PLEX): '', + } + HANDLERS = MediaServerType.handlers_dict() + + server_type = db.models.CharField( + _('server type'), + max_length=1, + db_index=True, + choices=MediaServerType.choices, + default=MediaServerType.PLEX, + help_text=_('Server type'), + ) + host = db.models.CharField( + _('host'), + db_index=True, + max_length=200, + help_text=_('Hostname or IP address of the media server'), + ) + port = db.models.PositiveIntegerField( + _('port'), + db_index=True, + help_text=_('Port number of the media server'), + ) + use_https = db.models.BooleanField( + _('use https'), + default=False, + help_text=_('Connect to the media server over HTTPS'), + ) + verify_https = db.models.BooleanField( + _('verify https'), + default=True, + help_text=_('If connecting over HTTPS, verify the SSL certificate is valid'), + ) + options = db.models.JSONField( + _('options'), + blank=False, + null=True, + help_text=_('JSON encoded options for the media server'), + ) + + def __str__(self): + return f'{self.get_server_type_display()} server at {self.url}' + + class Meta: + verbose_name = _('Media Server') + verbose_name_plural = _('Media Servers') + unique_together = ( + ('host', 'port'), + ) + + @property + def url(self): + scheme = 'https' if self.use_https else 'http' + return f'{scheme}://{self.host.strip()}:{self.port}' + + @property + def icon(self): + return self.ICONS.get(self.server_type) + + @property + def handler(self): + handler_class = self.HANDLERS.get(self.server_type) + return handler_class(self) + + @property + def loaded_options(self): + return self.options or dict() + + def validate(self): + return self.handler.validate() + + def update(self): + return self.handler.update() + + def get_help_html(self): + return self.handler.HELP From 83ecb14b88e6447560a81a41708c33ab1945b9bb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 21:41:29 -0400 Subject: [PATCH 084/174] Update media_server.py --- tubesync/sync/models/media_server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index 5e5e8c30..bb9424c4 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1,4 +1,3 @@ -import json from django import db from django.utils.translation import gettext_lazy as _ from ..choices import Val, MediaServerType @@ -48,7 +47,7 @@ class MediaServer(db.models.Model): _('options'), blank=False, null=True, - help_text=_('JSON encoded options for the media server'), + help_text=_('Options for the media server'), ) def __str__(self): From cfe2e30f17cbb16c4a06bd0a4bf58c2c6bd4bdff Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:02:06 -0400 Subject: [PATCH 085/174] Update views.py --- tubesync/sync/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 692675a0..4d376165 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1146,14 +1146,14 @@ class AddMediaServerView(FormView): def form_valid(self, form): # Assign mandatory fields, bundle other fields into options mediaserver = MediaServer(server_type=self.server_type) - options = {} + options = dict() model_fields = [field.name for field in MediaServer._meta.fields] for field_name, field_value in form.cleaned_data.items(): if field_name in model_fields: setattr(mediaserver, field_name, field_value) else: options[field_name] = field_value - mediaserver.options = json.dumps(options) + mediaserver.options = options # Test the media server details are valid try: mediaserver.validate() @@ -1260,21 +1260,21 @@ class UpdateMediaServerView(FormView, SingleObjectMixin): for field in self.object._meta.fields: if field.name in self.form_class.declared_fields: initial[field.name] = getattr(self.object, field.name) - for option_key, option_val in self.object.loaded_options.items(): + for option_key, option_val in self.object.options.items(): if option_key in self.form_class.declared_fields: initial[option_key] = option_val return initial def form_valid(self, form): # Assign mandatory fields, bundle other fields into options - options = {} + options = dict() model_fields = [field.name for field in MediaServer._meta.fields] for field_name, field_value in form.cleaned_data.items(): if field_name in model_fields: setattr(self.object, field_name, field_value) else: options[field_name] = field_value - self.object.options = json.dumps(options) + self.object.options = options # Test the media server details are valid try: self.object.validate() From a12363006980f09eb25206e605c833780e5c85ab Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:05:36 -0400 Subject: [PATCH 086/174] Update views.py --- tubesync/sync/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 4d376165..762f770e 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1,6 +1,5 @@ import glob import os -import json from base64 import b64decode import pathlib import sys From 84298ab5e5f3ec8d1ac6cf393c3e4d7f89c7005f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:12:24 -0400 Subject: [PATCH 087/174] Update mediaserver.html --- tubesync/sync/templates/sync/mediaserver.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/mediaserver.html b/tubesync/sync/templates/sync/mediaserver.html index 23546eba..2626d77c 100644 --- a/tubesync/sync/templates/sync/mediaserver.html +++ b/tubesync/sync/templates/sync/mediaserver.html @@ -28,7 +28,7 @@ Verify HTTPS Verify HTTPS
{% if mediaserver.verify_https %}{% else %}{% endif %} - {% for name, value in mediaserver.loaded_options.items %} + {% for name, value in mediaserver.options.items %} {{ name|title }} {{ name|title }}
{% if name in private_options %}{{ value|truncatechars:6 }} (hidden){% else %}{{ value }}{% endif %} From ea4733f725d0cebf0de2291f44432c8535deb851 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:31:12 -0400 Subject: [PATCH 088/174] Update mediaservers.py --- tubesync/sync/mediaservers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index e0f9e7e7..ceab239f 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -29,7 +29,7 @@ class MediaServer: def make_request_args(self, uri='/', token_header=None, headers={}, token_param=None, params={}): base_parts = urlsplit(self.object.url) if self.token is None: - self.token = self.object.loaded_options['token'] or None + self.token = self.object.options['token'] or None if token_header and self.token: headers.update({token_header: self.token}) self.headers.update(headers) @@ -116,7 +116,7 @@ class PlexMediaServer(MediaServer): if port < 1 or port > 65535: raise ValidationError('Plex Media Server "port" must be between 1 ' 'and 65535') - options = self.object.loaded_options + options = self.object.options if 'token' not in options: raise ValidationError('Plex Media Server requires a "token"') token = options['token'].strip() @@ -183,7 +183,7 @@ class PlexMediaServer(MediaServer): def update(self): # For each section / library ID pop off a request to refresh it - libraries = self.object.loaded_options.get('libraries', '') + libraries = self.object.options.get('libraries', '') for library_id in libraries.split(','): library_id = library_id.strip() uri = f'/library/sections/{library_id}/refresh' @@ -258,7 +258,7 @@ class JellyfinMediaServer(MediaServer): except (TypeError, ValueError): raise ValidationError('Jellyfin Media Server "port" must be an integer') - options = self.object.loaded_options + options = self.object.options if 'token' not in options: raise ValidationError('Jellyfin Media Server requires a "token"') if 'libraries' not in options: @@ -302,7 +302,7 @@ class JellyfinMediaServer(MediaServer): return True def update(self): - libraries = self.object.loaded_options.get('libraries', '').split(',') + libraries = self.object.options.get('libraries', '').split(',') for library_id in map(str.strip, libraries): uri = f'/Items/{library_id}/Refresh' response = self.make_request(uri, method='POST') From 5c987f1d582c4a222077c1a09a632f8c42ee8332 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:43:26 -0400 Subject: [PATCH 089/174] Update __init__.py --- tubesync/sync/models/__init__.py | 99 ++------------------------------ 1 file changed, 6 insertions(+), 93 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index a5e3d675..d0029c53 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -21,20 +21,21 @@ from common.logger import log from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) -from .youtube import ( get_media_info as get_youtube_media_info, +from ..youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) -from .utils import (seconds_to_timestr, parse_media_format, filter_response, +from ..utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote, multi_key_sort) -from .matching import ( get_best_combined_format, get_best_audio_format, +from ..matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) -from .fields import CommaSepChoiceField -from .choices import ( Val, CapChoices, Fallback, FileExtension, +from ..fields import CommaSepChoiceField +from ..choices import ( Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) @@ -1998,91 +1999,3 @@ class MetadataFormat(models.Model): self.site, self.value.get('format') or self.value.get('format_id'), ) - - -class MediaServer(models.Model): - ''' - A remote media server, such as a Plex server. - ''' - - ICONS = { - Val(MediaServerType.JELLYFIN): '', - Val(MediaServerType.PLEX): '', - } - HANDLERS = MediaServerType.handlers_dict() - - server_type = models.CharField( - _('server type'), - max_length=1, - db_index=True, - choices=MediaServerType.choices, - default=MediaServerType.PLEX, - help_text=_('Server type') - ) - host = models.CharField( - _('host'), - db_index=True, - max_length=200, - help_text=_('Hostname or IP address of the media server') - ) - port = models.PositiveIntegerField( - _('port'), - db_index=True, - help_text=_('Port number of the media server') - ) - use_https = models.BooleanField( - _('use https'), - default=False, - help_text=_('Connect to the media server over HTTPS') - ) - verify_https = models.BooleanField( - _('verify https'), - default=True, - help_text=_('If connecting over HTTPS, verify the SSL certificate is valid') - ) - options = models.TextField( - _('options'), - blank=False, # valid JSON only - null=True, - help_text=_('JSON encoded options for the media server') - ) - - def __str__(self): - return f'{self.get_server_type_display()} server at {self.url}' - - class Meta: - verbose_name = _('Media Server') - verbose_name_plural = _('Media Servers') - unique_together = ( - ('host', 'port'), - ) - - @property - def url(self): - scheme = 'https' if self.use_https else 'http' - return f'{scheme}://{self.host.strip()}:{self.port}' - - @property - def icon(self): - return self.ICONS.get(self.server_type) - - @property - def handler(self): - handler_class = self.HANDLERS.get(self.server_type) - return handler_class(self) - - @property - def loaded_options(self): - try: - return json.loads(self.options) - except Exception as e: - return {} - - def validate(self): - return self.handler.validate() - - def update(self): - return self.handler.update() - - def get_help_html(self): - return self.handler.HELP From ded3602d541c5ea93add1d0da07615c0336fc00b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:46:53 -0400 Subject: [PATCH 090/174] Update media_server.py --- tubesync/sync/models/media_server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index bb9424c4..a8ecefc6 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -74,10 +74,6 @@ class MediaServer(db.models.Model): handler_class = self.HANDLERS.get(self.server_type) return handler_class(self) - @property - def loaded_options(self): - return self.options or dict() - def validate(self): return self.handler.validate() From 5ab4e41b8d5918c9b01c60d790af7b902d0562fa Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:02:08 -0400 Subject: [PATCH 091/174] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 74 +++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index 8b137891..d44ca2a3 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -1 +1,75 @@ +import uuid +from django import db +from django.utils.translation import gettext_lazy as _ +#from .metadata import Metadata +from . import Metadata, JSONEncoder +class MetadataFormat(db.models.Model): + ''' + A format from the Metadata for an indexed `Media` item. + ''' + class Meta: + db_table = f'{Metadata._meta.db_table}_format' + verbose_name = _('Format from Media Metadata') + verbose_name_plural = _('Formats from Media Metadata') + unique_together = ( + ('metadata', 'site', 'key', 'number'), + ) + ordering = ['site', 'key', 'number'] + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the format'), + ) + metadata = db.models.ForeignKey( + Metadata, + # on_delete=models.DO_NOTHING, + on_delete=db.models.CASCADE, + related_name='format', + help_text=_('Metadata the format belongs to'), + null=False, + ) + site = db.models.CharField( + _('site'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='Youtube', + help_text=_('Site from which the format is available'), + ) + key = db.models.CharField( + _('key'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='', + help_text=_('Media identifier at the site from which this format is available'), + ) + number = db.models.PositiveIntegerField( + _('number'), + blank=False, + null=False, + help_text=_('Ordering number for this format'), + ) + value = db.models.JSONField( + _('value'), + encoder=JSONEncoder, + null=False, + default=dict, + help_text=_('JSON metadata format object'), + ) + + + def __str__(self): + template = '#{:n} "{}" from {}: {}' + return template.format( + self.number, + self.key, + self.site, + self.value.get('format') or self.value.get('format_id'), + ) From a00d760f0448e6aa3c15b8907120025e8198fc4e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:07:12 -0400 Subject: [PATCH 092/174] Create json.py --- tubesync/common/json.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tubesync/common/json.py diff --git a/tubesync/common/json.py b/tubesync/common/json.py new file mode 100644 index 00000000..e8a22e1c --- /dev/null +++ b/tubesync/common/json.py @@ -0,0 +1,16 @@ +from django.core.serializers.json import DjangoJSONEncoder + + +class JSONEncoder(DjangoJSONEncoder): + item_separator = ',' + key_separator = ':' + + def default(self, obj): + try: + iterable = iter(obj) + except TypeError: + pass + else: + return list(iterable) + return super().default(obj) + From d962d57ce08b7ac69a5dba87f14614adf8896a96 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:10:14 -0400 Subject: [PATCH 093/174] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index d44ca2a3..f4bea7ed 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -1,8 +1,9 @@ import uuid +from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ #from .metadata import Metadata -from . import Metadata, JSONEncoder +from . import Metadata class MetadataFormat(db.models.Model): ''' From 705169746be071a0b6d98ea7c455ec9f39194b62 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:12:28 -0400 Subject: [PATCH 094/174] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 8b137891..06340ed6 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -1 +1 @@ - +from . import Metadata From ac0eaa3ec5871d75edbf513bf252147822c49adf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:12:58 -0400 Subject: [PATCH 095/174] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index f4bea7ed..c116575b 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -2,8 +2,7 @@ import uuid from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ -#from .metadata import Metadata -from . import Metadata +from .metadata import Metadata class MetadataFormat(db.models.Model): ''' From 108f7169e33ebd23eb1602fedb6e768ade3c9704 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:17:38 -0400 Subject: [PATCH 096/174] Update __init__.py --- tubesync/sync/models/__init__.py | 72 +------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index d0029c53..8218b0cf 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -18,6 +18,7 @@ from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log +from common.json import JSONEncoder from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) @@ -35,6 +36,7 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .metadata_format import MetadataFormat from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') @@ -1929,73 +1931,3 @@ class Metadata(models.Model): return self.with_formats - -class MetadataFormat(models.Model): - ''' - A format from the Metadata for an indexed `Media` item. - ''' - class Meta: - db_table = f'{Metadata._meta.db_table}_format' - verbose_name = _('Format from Media Metadata') - verbose_name_plural = _('Formats from Media Metadata') - unique_together = ( - ('metadata', 'site', 'key', 'number'), - ) - ordering = ['site', 'key', 'number'] - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the format'), - ) - metadata = models.ForeignKey( - Metadata, - # on_delete=models.DO_NOTHING, - on_delete=models.CASCADE, - related_name='format', - help_text=_('Metadata the format belongs to'), - null=False, - ) - site = models.CharField( - _('site'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='Youtube', - help_text=_('Site from which the format is available'), - ) - key = models.CharField( - _('key'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='', - help_text=_('Media identifier at the site from which this format is available'), - ) - number = models.PositiveIntegerField( - _('number'), - blank=False, - null=False, - help_text=_('Ordering number for this format') - ) - value = models.JSONField( - _('value'), - encoder=JSONEncoder, - null=False, - default=dict, - help_text=_('JSON metadata format object'), - ) - - - def __str__(self): - template = '#{:n} "{}" from {}: {}' - return template.format( - self.number, - self.key, - self.site, - self.value.get('format') or self.value.get('format_id'), - ) From 9e66bf5151d3164ab77cf2d3d57afc702eea7786 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:37:27 -0400 Subject: [PATCH 097/174] Update metadata.py --- tubesync/sync/models/metadata.py | 155 ++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 06340ed6..de09ca56 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -1 +1,154 @@ -from . import Metadata +import uuid +from common.json import JSONEncoder +from common.timestamp import timestamp_to_datetime +from common.utils import django_queryset_generator as qs_gen +from django import db +from django.utils.translation import gettext_lazy as _ +#from .media import Media +from . import Media + + +class Metadata(db.models.Model): + ''' + Metadata for an indexed `Media` item. + ''' + class Meta: + db_table = 'sync_media_metadata' + verbose_name = _('Metadata about Media') + verbose_name_plural = _('Metadata about Media') + unique_together = ( + ('media', 'site', 'key'), + ) + get_latest_by = ["-retrieved", "-created"] + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the metadata'), + ) + media = db.models.OneToOneField( + Media, + # on_delete=models.DO_NOTHING, + on_delete=db.models.SET_NULL, + related_name='new_metadata', + help_text=_('Media the metadata belongs to'), + null=True, + parent_link=False, + ) + site = db.models.CharField( + _('site'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='Youtube', + help_text=_('Site from which the metadata was retrieved'), + ) + key = db.models.CharField( + _('key'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='', + help_text=_('Media identifier at the site from which the metadata was retrieved'), + ) + created = db.models.DateTimeField( + _('created'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was created'), + ) + retrieved = db.models.DateTimeField( + _('retrieved'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was retrieved'), + ) + uploaded = db.models.DateTimeField( + _('uploaded'), + db_index=True, + null=True, + help_text=_('Date and time the media was uploaded'), + ) + published = db.models.DateTimeField( + _('published'), + db_index=True, + null=True, + help_text=_('Date and time the media was published'), + ) + value = db.models.JSONField( + _('value'), + encoder=JSONEncoder, + null=False, + default=dict, + help_text=_('JSON metadata object'), + ) + + + def __str__(self): + template = '"{}" from {} at: {}' + return template.format( + self.key, + self.site, + self.retrieved.isoformat(timespec='seconds'), + ) + + @db.transaction.atomic(durable=False) + def ingest_formats(self, formats=list(), /): + number = 0 + for number, format in enumerate(formats, start=1): + mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) + mdf.value = format + mdf.save() + if number > 0: + # delete any numbers we did not overwrite or create + self.format.filter(site=self.site, key=self.key, number__gt=number).delete() + + @property + def with_formats(self): + formats = self.format.all().order_by('number') + formats_list = [ f.value for f in qs_gen(formats) ] + metadata = self.value.copy() + metadata.update(dict(formats=formats_list)) + return metadata + + @db.transaction.atomic(durable=False) + def ingest_metadata(self, data): + assert isinstance(data, dict), type(data) + + try: + self.retrieved = timestamp_to_datetime( + self.media.get_metadata_first_value( + 'epoch', + arg_dict=data, + ) + ) or self.created + except AssertionError: + self.retrieved = self.created + + try: + self.published = timestamp_to_datetime( + self.media.get_metadata_first_value( + ('release_timestamp', 'timestamp',), + arg_dict=data, + ) + ) or self.media.published + except AssertionError: + self.published = self.media.published + + self.value = data.copy() # try not to have side-effects for the caller + formats_key = self.media.get_metadata_field('formats') + formats = self.value.pop(formats_key, list()) + self.uploaded = min( + self.published, + self.retrieved, + self.media.created, + ) + self.save() + self.ingest_formats(formats) + + return self.with_formats + From 0d9542163bf1d64f1b65b36886a071a645a911c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:43:16 -0400 Subject: [PATCH 098/174] Update __init__.py --- tubesync/sync/models/__init__.py | 161 +------------------------------ 1 file changed, 1 insertion(+), 160 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 8218b0cf..7ff69e97 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -36,26 +36,13 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) -class JSONEncoder(DjangoJSONEncoder): - item_separator = ',' - key_separator = ':' - - def default(self, obj): - try: - iterable = iter(obj) - except TypeError: - pass - else: - return list(iterable) - return super().default(obj) - - class Source(models.Model): ''' A Source is a source of media. Currently, this is either a YouTube channel @@ -1785,149 +1772,3 @@ class Media(models.Model): except OSError as e: pass - -class Metadata(models.Model): - ''' - Metadata for an indexed `Media` item. - ''' - class Meta: - db_table = 'sync_media_metadata' - verbose_name = _('Metadata about Media') - verbose_name_plural = _('Metadata about Media') - unique_together = ( - ('media', 'site', 'key'), - ) - get_latest_by = ["-retrieved", "-created"] - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the metadata'), - ) - media = models.OneToOneField( - Media, - # on_delete=models.DO_NOTHING, - on_delete=models.SET_NULL, - related_name='new_metadata', - help_text=_('Media the metadata belongs to'), - null=True, - parent_link=False, - ) - site = models.CharField( - _('site'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='Youtube', - help_text=_('Site from which the metadata was retrieved'), - ) - key = models.CharField( - _('key'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='', - help_text=_('Media identifier at the site from which the metadata was retrieved'), - ) - created = models.DateTimeField( - _('created'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the metadata was created'), - ) - retrieved = models.DateTimeField( - _('retrieved'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the metadata was retrieved'), - ) - uploaded = models.DateTimeField( - _('uploaded'), - db_index=True, - null=True, - help_text=_('Date and time the media was uploaded'), - ) - published = models.DateTimeField( - _('published'), - db_index=True, - null=True, - help_text=_('Date and time the media was published'), - ) - value = models.JSONField( - _('value'), - encoder=JSONEncoder, - null=False, - default=dict, - help_text=_('JSON metadata object'), - ) - - - def __str__(self): - template = '"{}" from {} at: {}' - return template.format( - self.key, - self.site, - self.retrieved.isoformat(timespec='seconds'), - ) - - @atomic(durable=False) - def ingest_formats(self, formats=list(), /): - number = 0 - for number, format in enumerate(formats, start=1): - mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) - mdf.value = format - mdf.save() - if number > 0: - # delete any numbers we did not overwrite or create - self.format.filter(site=self.site, key=self.key, number__gt=number).delete() - - @property - def with_formats(self): - formats = self.format.all().order_by('number') - formats_list = [ f.value for f in qs_gen(formats) ] - metadata = self.value.copy() - metadata.update(dict(formats=formats_list)) - return metadata - - @atomic(durable=False) - def ingest_metadata(self, data): - assert isinstance(data, dict), type(data) - from common.timestamp import timestamp_to_datetime - - try: - self.retrieved = timestamp_to_datetime( - self.media.get_metadata_first_value( - 'epoch', - arg_dict=data, - ) - ) or self.created - except AssertionError: - self.retrieved = self.created - - try: - self.published = timestamp_to_datetime( - self.media.get_metadata_first_value( - ('release_timestamp', 'timestamp',), - arg_dict=data, - ) - ) or self.media.published - except AssertionError: - self.published = self.media.published - - self.value = data.copy() # try not to have side-effects for the caller - formats_key = self.media.get_metadata_field('formats') - formats = self.value.pop(formats_key, list()) - self.uploaded = min( - self.published, - self.retrieved, - self.media.created, - ) - self.save() - self.ingest_formats(formats) - - return self.with_formats - From a5afdc6a456b998120a8a9ea2883388ac9b5c000 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:46:09 -0400 Subject: [PATCH 099/174] Update metadata.py --- tubesync/sync/models/metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index de09ca56..17d214fb 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,8 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -#from .media import Media -from . import Media +from .media import Media class Metadata(db.models.Model): From 6cc19363257791682a7f6f74d9e4ebba2f9cc6ad Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:48:43 -0400 Subject: [PATCH 100/174] Rename __init__.py to legacy.py --- tubesync/sync/models/{__init__.py => legacy.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/models/{__init__.py => legacy.py} (100%) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/legacy.py similarity index 100% rename from tubesync/sync/models/__init__.py rename to tubesync/sync/models/legacy.py From 2e59339c6bdeea272fe253a0fbc68023f6c42434 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:50:01 -0400 Subject: [PATCH 101/174] Create __init__.py --- tubesync/sync/models/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tubesync/sync/models/__init__.py diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py new file mode 100644 index 00000000..b144173f --- /dev/null +++ b/tubesync/sync/models/__init__.py @@ -0,0 +1,5 @@ +from .metadata import Metadata +from .metadata_format import MetadataFormat +from .media_server import MediaServer + +from .legacy import * From 0251b9a8aa1ab44363f718c99548df5640ed8bd8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:50:36 -0400 Subject: [PATCH 102/174] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 17d214fb..e062ea0c 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,7 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -from .media import Media +from .legacy import Media class Metadata(db.models.Model): From d4003416d20960b8d911c7aa9c59795351dbaac2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:53:33 -0400 Subject: [PATCH 103/174] Update legacy.py --- tubesync/sync/models/legacy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/legacy.py index 7ff69e97..a7facceb 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/legacy.py @@ -36,9 +36,6 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) -from .metadata import Metadata -from .metadata_format import MetadataFormat -from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) From aad8709bb8c27acd6ddc918656c7ec542dd6f087 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:56:38 -0400 Subject: [PATCH 104/174] Delete tubesync/sync/models/media.py --- tubesync/sync/models/media.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tubesync/sync/models/media.py diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py deleted file mode 100644 index 8b137891..00000000 --- a/tubesync/sync/models/media.py +++ /dev/null @@ -1 +0,0 @@ - From 9ceafe9fd2658a33415921d289bea9753e41ee68 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:21:26 -0400 Subject: [PATCH 105/174] Update source.py --- tubesync/sync/models/source.py | 546 +++++++++++++++++++++++++++++++++ 1 file changed, 546 insertions(+) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 8b137891..01a0d36e 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -1 +1,547 @@ +import os +import re +import uuid +from pathlib import Path +from django import db +from django.conf import settings +from django.core.exceptions import SuspiciousOperation +from django.utils import timezone +from django.utils.text import slugify +from django.utils.translation import gettext_lazy as _ +from ..choices import (Val, + SponsorBlock_Category, YouTube_SourceType, IndexSchedule, + CapChoices, FilterSeconds, FileExtension, + SourceResolution, SourceResolutionInteger, + YouTube_VideoCodec, YouTube_AudioCodec, +) +from ..fields import CommaSepChoiceField +from ..youtube import ( + get_media_info as get_youtube_media_info, + get_channel_image_info as get_youtube_channel_image_info, +) +from .legacy import _srctype_dict, media_file_storage + + +class Source(db.models.Model): + ''' + A Source is a source of media. Currently, this is either a YouTube channel + or a YouTube playlist. + ''' + + sponsorblock_categories = CommaSepChoiceField( + _(''), + max_length=128, + possible_choices=SponsorBlock_Category.choices, + all_choice='all', + allow_all=True, + all_label='(All Categories)', + default='all', + help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'), + ) + embed_metadata = db.models.BooleanField( + _('embed metadata'), + default=False, + help_text=_('Embed metadata from source into file'), + ) + embed_thumbnail = db.models.BooleanField( + _('embed thumbnail'), + default=False, + help_text=_('Embed thumbnail into the file'), + ) + enable_sponsorblock = db.models.BooleanField( + _('enable sponsorblock'), + default=True, + help_text=_('Use SponsorBlock?'), + ) + + # Fontawesome icons used for the source on the front end + ICONS = _srctype_dict('') + + # Format to use to display a URL for the source + URLS = dict(zip( + YouTube_SourceType.values, + ( + 'https://www.youtube.com/c/{key}', + 'https://www.youtube.com/channel/{key}', + 'https://www.youtube.com/playlist?list={key}', + ), + )) + + # Format used to create indexable URLs + INDEX_URLS = dict(zip( + YouTube_SourceType.values, + ( + 'https://www.youtube.com/c/{key}/{type}', + 'https://www.youtube.com/channel/{key}/{type}', + 'https://www.youtube.com/playlist?list={key}', + ), + )) + + # Callback functions to get a list of media from the source + INDEXERS = _srctype_dict(get_youtube_media_info) + + # Field names to find the media ID used as the key when storing media + KEY_FIELD = _srctype_dict('id') + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the source'), + ) + created = db.models.DateTimeField( + _('created'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the source was created'), + ) + last_crawl = db.models.DateTimeField( + _('last crawl'), + db_index=True, + null=True, + blank=True, + help_text=_('Date and time the source was last crawled'), + ) + source_type = db.models.CharField( + _('source type'), + max_length=1, + db_index=True, + choices=YouTube_SourceType.choices, + default=YouTube_SourceType.CHANNEL, + help_text=_('Source type'), + ) + key = db.models.CharField( + _('key'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Source key, such as exact YouTube channel name or playlist ID'), + ) + name = db.models.CharField( + _('name'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Friendly name for the source, used locally in TubeSync only'), + ) + directory = db.models.CharField( + _('directory'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Directory name to save the media into'), + ) + media_format = db.models.CharField( + _('media format'), + max_length=200, + default=settings.MEDIA_FORMATSTR_DEFAULT, + help_text=_('File format to use for saving files, detailed options at bottom of page.'), + ) + index_schedule = db.models.IntegerField( + _('index schedule'), + choices=IndexSchedule.choices, + db_index=True, + default=IndexSchedule.EVERY_24_HOURS, + help_text=_('Schedule of how often to index the source for new media'), + ) + download_media = db.models.BooleanField( + _('download media'), + default=True, + help_text=_('Download media from this source, if not selected the source will only be indexed'), + ) + index_videos = db.models.BooleanField( + _('index videos'), + default=True, + help_text=_('Index video media from this source'), + ) + index_streams = db.models.BooleanField( + _('index streams'), + default=False, + help_text=_('Index live stream media from this source'), + ) + download_cap = db.models.IntegerField( + _('download cap'), + choices=CapChoices.choices, + default=CapChoices.CAP_NOCAP, + help_text=_('Do not download media older than this capped date'), + ) + delete_old_media = db.models.BooleanField( + _('delete old media'), + default=False, + help_text=_('Delete old media after "days to keep" days?'), + ) + days_to_keep = db.models.PositiveSmallIntegerField( + _('days to keep'), + default=14, + help_text=_( + 'If "delete old media" is ticked, the number of days after which ' + 'to automatically delete media' + ), + ) + filter_text = db.models.CharField( + _('filter string'), + max_length=200, + default='', + blank=True, + help_text=_('Regex compatible filter string for video titles'), + ) + filter_text_invert = db.models.BooleanField( + _('invert filter text matching'), + default=False, + help_text=_('Invert filter string regex match, skip any matching titles when selected'), + ) + filter_seconds = db.models.PositiveIntegerField( + _('filter seconds'), + blank=True, + null=True, + help_text=_('Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering'), + ) + filter_seconds_min = db.models.BooleanField( + _('filter seconds min/max'), + choices=FilterSeconds.choices, + default=Val(FilterSeconds.MIN), + help_text=_( + 'When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video ' + 'greater than maximum) video duration' + ), + ) + delete_removed_media = db.models.BooleanField( + _('delete removed media'), + default=False, + help_text=_('Delete media that is no longer on this playlist'), + ) + delete_files_on_disk = db.models.BooleanField( + _('delete files on disk'), + default=False, + help_text=_('Delete files on disk when they are removed from TubeSync'), + ) + source_resolution = db.models.CharField( + _('source resolution'), + max_length=8, + db_index=True, + choices=SourceResolution.choices, + default=SourceResolution.VIDEO_1080P, + help_text=_('Source resolution, desired video resolution to download'), + ) + source_vcodec = db.models.CharField( + _('source video codec'), + max_length=8, + db_index=True, + choices=YouTube_VideoCodec.choices, + default=YouTube_VideoCodec.VP9, + help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)'), + ) + source_acodec = db.models.CharField( + _('source audio codec'), + max_length=8, + db_index=True, + choices=YouTube_AudioCodec.choices, + default=YouTube_AudioCodec.OPUS, + help_text=_('Source audio codec, desired audio encoding format to download'), + ) + prefer_60fps = db.models.BooleanField( + _('prefer 60fps'), + default=True, + help_text=_('Where possible, prefer 60fps media for this source'), + ) + prefer_hdr = db.models.BooleanField( + _('prefer hdr'), + default=False, + help_text=_('Where possible, prefer HDR media for this source'), + ) + fallback = db.models.CharField( + _('fallback'), + max_length=1, + db_index=True, + choices=Fallback.choices, + default=Fallback.NEXT_BEST_HD, + help_text=_('What do do when media in your source resolution and codecs is not available'), + ) + copy_channel_images = db.models.BooleanField( + _('copy channel images'), + default=False, + help_text=_('Copy channel banner and avatar. These may be detected and used by some media servers'), + ) + copy_thumbnails = db.models.BooleanField( + _('copy thumbnails'), + default=False, + help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers'), + ) + write_nfo = db.models.BooleanField( + _('write nfo'), + default=False, + help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers'), + ) + write_json = db.models.BooleanField( + _('write json'), + default=False, + help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers'), + ) + has_failed = db.models.BooleanField( + _('has failed'), + default=False, + help_text=_('Source has failed to index media'), + ) + + write_subtitles = db.models.BooleanField( + _('write subtitles'), + default=False, + help_text=_('Download video subtitles'), + ) + + auto_subtitles = db.models.BooleanField( + _('accept auto-generated subs'), + default=False, + help_text=_('Accept auto-generated subtitles'), + ) + sub_langs = db.models.CharField( + _('subs langs'), + max_length=30, + default='en', + help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), + validators=[ + RegexValidator( + regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", + message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat'), + ), + ], + ) + + def __str__(self): + return self.name + + class Meta: + verbose_name = _('Source') + verbose_name_plural = _('Sources') + + @property + def icon(self): + return self.ICONS.get(self.source_type) + + @property + def slugname(self): + replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') + return slugify(replaced)[:80] + + def deactivate(self): + self.download_media = False + self.index_streams = False + self.index_videos = False + self.index_schedule = IndexSchedule.NEVER + self.save(update_fields={ + 'download_media', + 'index_streams', + 'index_videos', + 'index_schedule', + }) + + @property + def is_active(self): + active = ( + self.download_media or + self.index_streams or + self.index_videos + ) + return self.index_schedule and active + + @property + def is_audio(self): + return self.source_resolution == SourceResolution.AUDIO.value + + @property + def is_playlist(self): + return self.source_type == YouTube_SourceType.PLAYLIST.value + + @property + def is_video(self): + return not self.is_audio + + @property + def download_cap_date(self): + delta = self.download_cap + if delta > 0: + return timezone.now() - timezone.timedelta(seconds=delta) + else: + return False + + @property + def days_to_keep_date(self): + delta = self.days_to_keep + if delta > 0: + return timezone.now() - timezone.timedelta(days=delta) + else: + return False + + @property + def extension(self): + ''' + The extension is also used by youtube-dl to set the output container. As + it is possible to quite easily pick combinations of codecs and containers + which are invalid (e.g. OPUS audio in an MP4 container) just set this for + people. All video is set to mkv containers, audio-only is set to m4a or ogg + depending on audio codec. + ''' + if self.is_audio: + if self.source_acodec == Val(YouTube_AudioCodec.MP4A): + return Val(FileExtension.M4A) + elif self.source_acodec == Val(YouTube_AudioCodec.OPUS): + return Val(FileExtension.OGG) + else: + raise ValueError('Unable to choose audio extension, uknown acodec') + else: + return Val(FileExtension.MKV) + + @classmethod + def create_url(cls, source_type, key): + url = cls.URLS.get(source_type) + return url.format(key=key) + + @classmethod + def create_index_url(cls, source_type, key, type): + url = cls.INDEX_URLS.get(source_type) + return url.format(key=key, type=type) + + @property + def url(self): + return self.__class__.create_url(self.source_type, self.key) + + def get_index_url(self, type): + return self.__class__.create_index_url(self.source_type, self.key, type) + + @property + def format_summary(self): + if self.is_audio: + vc = 'none' + else: + vc = self.source_vcodec + ac = self.source_acodec + f = ' 60FPS' if self.is_video and self.prefer_60fps else '' + h = ' HDR' if self.is_video and self.prefer_hdr else '' + return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip() + + @property + def directory_path(self): + download_dir = Path(media_file_storage.location) + return download_dir / self.type_directory_path + + @property + def type_directory_path(self): + if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: + if self.is_audio: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + else: + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + else: + return Path(self.directory) + + def make_directory(self): + return os.makedirs(self.directory_path, exist_ok=True) + + @property + def get_image_url(self): + if self.is_playlist: + raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') + + return get_youtube_channel_image_info(self.url) + + + def directory_exists(self): + return (os.path.isdir(self.directory_path) and + os.access(self.directory_path, os.W_OK)) + + @property + def key_field(self): + return self.KEY_FIELD.get(self.source_type, '') + + @property + def source_resolution_height(self): + return SourceResolutionInteger.get(self.source_resolution, 0) + + @property + def can_fallback(self): + return self.fallback != Val(Fallback.FAIL) + + @property + def example_media_format_dict(self): + ''' + Populates a dict with real-ish and some placeholder data for media name + format strings. Used for example filenames and media_format validation. + ''' + fmt = [] + if self.source_resolution: + fmt.append(self.source_resolution) + if self.source_vcodec: + fmt.append(self.source_vcodec.lower()) + if self.source_acodec: + fmt.append(self.source_acodec.lower()) + if self.prefer_60fps: + fmt.append('60fps') + if self.prefer_hdr: + fmt.append('hdr') + now = timezone.now() + return { + 'yyyymmdd': now.strftime('%Y%m%d'), + 'yyyy_mm_dd': now.strftime('%Y-%m-%d'), + 'yyyy': now.strftime('%Y'), + 'mm': now.strftime('%m'), + 'dd': now.strftime('%d'), + 'source': self.slugname, + 'source_full': self.name, + 'uploader': 'Some Channel Name', + 'title': 'some-media-title-name', + 'title_full': 'Some Media Title Name', + 'key': 'SoMeUnIqUiD', + 'format': '-'.join(fmt), + 'playlist_title': 'Some Playlist Title', + 'video_order': '01', + 'ext': self.extension, + 'resolution': self.source_resolution if self.source_resolution else '', + 'height': '720' if self.source_resolution else '', + 'width': '1280' if self.source_resolution else '', + 'vcodec': self.source_vcodec.lower() if self.source_vcodec else '', + 'acodec': self.source_acodec.lower(), + 'fps': '24' if self.source_resolution else '', + 'hdr': 'hdr' if self.source_resolution else '' + } + + def get_example_media_format(self): + try: + return self.media_format.format(**self.example_media_format_dict) + except Exception as e: + return '' + + def is_regex_match(self, media_item_title): + if not self.filter_text: + return True + return bool(re.search(self.filter_text, media_item_title)) + + def get_index(self, type): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + days = None + if self.download_cap_date: + days = timezone.timedelta(seconds=self.download_cap).days + response = indexer(self.get_index_url(type=type), days=days) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + + def index_media(self): + ''' + Index the media source returning a list of media metadata as dicts. + ''' + entries = list() + if self.index_videos: + entries += self.get_index('videos') + # Playlists do something different that I have yet to figure out + if not self.is_playlist: + if self.index_streams: + entries += self.get_index('streams') + + if settings.MAX_ENTRIES_PROCESSING: + entries = entries[:settings.MAX_ENTRIES_PROCESSING] + return entries From 4b42670824f927c1a9d01661855e3e57c2cd8c74 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:24:02 -0400 Subject: [PATCH 106/174] Update __init__.py --- tubesync/sync/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index b144173f..ff0f7b3b 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,3 +1,4 @@ +from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer From b5ee002404e98a2ef31c97286e9e3c6aa0f401dd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:29:34 -0400 Subject: [PATCH 107/174] Update legacy.py --- tubesync/sync/models/legacy.py | 519 --------------------------------- 1 file changed, 519 deletions(-) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/legacy.py index a7facceb..83bd9256 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/legacy.py @@ -40,525 +40,6 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) -class Source(models.Model): - ''' - A Source is a source of media. Currently, this is either a YouTube channel - or a YouTube playlist. - ''' - - sponsorblock_categories = CommaSepChoiceField( - _(''), - max_length=128, - possible_choices=SponsorBlock_Category.choices, - all_choice='all', - allow_all=True, - all_label='(All Categories)', - default='all', - help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.') - ) - embed_metadata = models.BooleanField( - _('embed metadata'), - default=False, - help_text=_('Embed metadata from source into file') - ) - embed_thumbnail = models.BooleanField( - _('embed thumbnail'), - default=False, - help_text=_('Embed thumbnail into the file') - ) - enable_sponsorblock = models.BooleanField( - _('enable sponsorblock'), - default=True, - help_text=_('Use SponsorBlock?') - ) - - # Fontawesome icons used for the source on the front end - ICONS = _srctype_dict('') - - # Format to use to display a URL for the source - URLS = dict(zip( - YouTube_SourceType.values, - ( - 'https://www.youtube.com/c/{key}', - 'https://www.youtube.com/channel/{key}', - 'https://www.youtube.com/playlist?list={key}', - ), - )) - - # Format used to create indexable URLs - INDEX_URLS = dict(zip( - YouTube_SourceType.values, - ( - 'https://www.youtube.com/c/{key}/{type}', - 'https://www.youtube.com/channel/{key}/{type}', - 'https://www.youtube.com/playlist?list={key}', - ), - )) - - # Callback functions to get a list of media from the source - INDEXERS = _srctype_dict(get_youtube_media_info) - - # Field names to find the media ID used as the key when storing media - KEY_FIELD = _srctype_dict('id') - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the source') - ) - created = models.DateTimeField( - _('created'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the source was created') - ) - last_crawl = models.DateTimeField( - _('last crawl'), - db_index=True, - null=True, - blank=True, - help_text=_('Date and time the source was last crawled') - ) - source_type = models.CharField( - _('source type'), - max_length=1, - db_index=True, - choices=YouTube_SourceType.choices, - default=YouTube_SourceType.CHANNEL, - help_text=_('Source type') - ) - key = models.CharField( - _('key'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Source key, such as exact YouTube channel name or playlist ID') - ) - name = models.CharField( - _('name'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Friendly name for the source, used locally in TubeSync only') - ) - directory = models.CharField( - _('directory'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Directory name to save the media into') - ) - media_format = models.CharField( - _('media format'), - max_length=200, - default=settings.MEDIA_FORMATSTR_DEFAULT, - help_text=_('File format to use for saving files, detailed options at bottom of page.') - ) - index_schedule = models.IntegerField( - _('index schedule'), - choices=IndexSchedule.choices, - db_index=True, - default=IndexSchedule.EVERY_24_HOURS, - help_text=_('Schedule of how often to index the source for new media') - ) - download_media = models.BooleanField( - _('download media'), - default=True, - help_text=_('Download media from this source, if not selected the source will only be indexed') - ) - index_videos = models.BooleanField( - _('index videos'), - default=True, - help_text=_('Index video media from this source') - ) - index_streams = models.BooleanField( - _('index streams'), - default=False, - help_text=_('Index live stream media from this source') - ) - download_cap = models.IntegerField( - _('download cap'), - choices=CapChoices.choices, - default=CapChoices.CAP_NOCAP, - help_text=_('Do not download media older than this capped date') - ) - delete_old_media = models.BooleanField( - _('delete old media'), - default=False, - help_text=_('Delete old media after "days to keep" days?') - ) - days_to_keep = models.PositiveSmallIntegerField( - _('days to keep'), - default=14, - help_text=_('If "delete old media" is ticked, the number of days after which ' - 'to automatically delete media') - ) - filter_text = models.CharField( - _('filter string'), - max_length=200, - default='', - blank=True, - help_text=_('Regex compatible filter string for video titles') - ) - filter_text_invert = models.BooleanField( - _("invert filter text matching"), - default=False, - help_text="Invert filter string regex match, skip any matching titles when selected", - ) - filter_seconds = models.PositiveIntegerField( - _('filter seconds'), - blank=True, - null=True, - help_text=_('Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering') - ) - filter_seconds_min = models.BooleanField( - _('filter seconds min/max'), - choices=FilterSeconds.choices, - default=Val(FilterSeconds.MIN), - help_text=_('When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video ' - 'greater than maximum) video duration') - ) - delete_removed_media = models.BooleanField( - _('delete removed media'), - default=False, - help_text=_('Delete media that is no longer on this playlist') - ) - delete_files_on_disk = models.BooleanField( - _('delete files on disk'), - default=False, - help_text=_('Delete files on disk when they are removed from TubeSync') - ) - source_resolution = models.CharField( - _('source resolution'), - max_length=8, - db_index=True, - choices=SourceResolution.choices, - default=SourceResolution.VIDEO_1080P, - help_text=_('Source resolution, desired video resolution to download') - ) - source_vcodec = models.CharField( - _('source video codec'), - max_length=8, - db_index=True, - choices=list(reversed(YouTube_VideoCodec.choices)), - default=YouTube_VideoCodec.VP9, - help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)') - ) - source_acodec = models.CharField( - _('source audio codec'), - max_length=8, - db_index=True, - choices=list(reversed(YouTube_AudioCodec.choices)), - default=YouTube_AudioCodec.OPUS, - help_text=_('Source audio codec, desired audio encoding format to download') - ) - prefer_60fps = models.BooleanField( - _('prefer 60fps'), - default=True, - help_text=_('Where possible, prefer 60fps media for this source') - ) - prefer_hdr = models.BooleanField( - _('prefer hdr'), - default=False, - help_text=_('Where possible, prefer HDR media for this source') - ) - fallback = models.CharField( - _('fallback'), - max_length=1, - db_index=True, - choices=Fallback.choices, - default=Fallback.NEXT_BEST_HD, - help_text=_('What do do when media in your source resolution and codecs is not available') - ) - copy_channel_images = models.BooleanField( - _('copy channel images'), - default=False, - help_text=_('Copy channel banner and avatar. These may be detected and used by some media servers') - ) - copy_thumbnails = models.BooleanField( - _('copy thumbnails'), - default=False, - help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers') - ) - write_nfo = models.BooleanField( - _('write nfo'), - default=False, - help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers') - ) - write_json = models.BooleanField( - _('write json'), - default=False, - help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers') - ) - has_failed = models.BooleanField( - _('has failed'), - default=False, - help_text=_('Source has failed to index media') - ) - - write_subtitles = models.BooleanField( - _('write subtitles'), - default=False, - help_text=_('Download video subtitles') - ) - - auto_subtitles = models.BooleanField( - _('accept auto-generated subs'), - default=False, - help_text=_('Accept auto-generated subtitles') - ) - sub_langs = models.CharField( - _('subs langs'), - max_length=30, - default='en', - help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), - validators=[ - RegexValidator( - regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", - message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') - ) - ] - ) - - def __str__(self): - return self.name - - class Meta: - verbose_name = _('Source') - verbose_name_plural = _('Sources') - - @property - def icon(self): - return self.ICONS.get(self.source_type) - - @property - def slugname(self): - replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') - return slugify(replaced)[:80] - - def deactivate(self): - self.download_media = False - self.index_streams = False - self.index_videos = False - self.index_schedule = IndexSchedule.NEVER - self.save(update_fields={ - 'download_media', - 'index_streams', - 'index_videos', - 'index_schedule', - }) - - @property - def is_active(self): - active = ( - self.download_media or - self.index_streams or - self.index_videos - ) - return self.index_schedule and active - - @property - def is_audio(self): - return self.source_resolution == SourceResolution.AUDIO.value - - @property - def is_playlist(self): - return self.source_type == YouTube_SourceType.PLAYLIST.value - - @property - def is_video(self): - return not self.is_audio - - @property - def download_cap_date(self): - delta = self.download_cap - if delta > 0: - return timezone.now() - timedelta(seconds=delta) - else: - return False - - @property - def days_to_keep_date(self): - delta = self.days_to_keep - if delta > 0: - return timezone.now() - timedelta(days=delta) - else: - return False - - @property - def extension(self): - ''' - The extension is also used by youtube-dl to set the output container. As - it is possible to quite easily pick combinations of codecs and containers - which are invalid (e.g. OPUS audio in an MP4 container) just set this for - people. All video is set to mkv containers, audio-only is set to m4a or ogg - depending on audio codec. - ''' - if self.is_audio: - if self.source_acodec == Val(YouTube_AudioCodec.MP4A): - return Val(FileExtension.M4A) - elif self.source_acodec == Val(YouTube_AudioCodec.OPUS): - return Val(FileExtension.OGG) - else: - raise ValueError('Unable to choose audio extension, uknown acodec') - else: - return Val(FileExtension.MKV) - - @classmethod - def create_url(obj, source_type, key): - url = obj.URLS.get(source_type) - return url.format(key=key) - - @classmethod - def create_index_url(obj, source_type, key, type): - url = obj.INDEX_URLS.get(source_type) - return url.format(key=key, type=type) - - @property - def url(self): - return Source.create_url(self.source_type, self.key) - - def get_index_url(self, type): - return Source.create_index_url(self.source_type, self.key, type) - - @property - def format_summary(self): - if self.is_audio: - vc = 'none' - else: - vc = self.source_vcodec - ac = self.source_acodec - f = ' 60FPS' if self.is_video and self.prefer_60fps else '' - h = ' HDR' if self.is_video and self.prefer_hdr else '' - return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip() - - @property - def directory_path(self): - download_dir = Path(media_file_storage.location) - return download_dir / self.type_directory_path - - @property - def type_directory_path(self): - if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: - if self.is_audio: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory - else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory - else: - return Path(self.directory) - - def make_directory(self): - return os.makedirs(self.directory_path, exist_ok=True) - - @property - def get_image_url(self): - if self.is_playlist: - raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') - - return get_youtube_channel_image_info(self.url) - - - def directory_exists(self): - return (os.path.isdir(self.directory_path) and - os.access(self.directory_path, os.W_OK)) - - @property - def key_field(self): - return self.KEY_FIELD.get(self.source_type, '') - - @property - def source_resolution_height(self): - return SourceResolutionInteger.get(self.source_resolution, 0) - - @property - def can_fallback(self): - return self.fallback != Val(Fallback.FAIL) - - @property - def example_media_format_dict(self): - ''' - Populates a dict with real-ish and some placeholder data for media name - format strings. Used for example filenames and media_format validation. - ''' - fmt = [] - if self.source_resolution: - fmt.append(self.source_resolution) - if self.source_vcodec: - fmt.append(self.source_vcodec.lower()) - if self.source_acodec: - fmt.append(self.source_acodec.lower()) - if self.prefer_60fps: - fmt.append('60fps') - if self.prefer_hdr: - fmt.append('hdr') - now = timezone.now() - return { - 'yyyymmdd': now.strftime('%Y%m%d'), - 'yyyy_mm_dd': now.strftime('%Y-%m-%d'), - 'yyyy': now.strftime('%Y'), - 'mm': now.strftime('%m'), - 'dd': now.strftime('%d'), - 'source': self.slugname, - 'source_full': self.name, - 'uploader': 'Some Channel Name', - 'title': 'some-media-title-name', - 'title_full': 'Some Media Title Name', - 'key': 'SoMeUnIqUiD', - 'format': '-'.join(fmt), - 'playlist_title': 'Some Playlist Title', - 'video_order': '01', - 'ext': self.extension, - 'resolution': self.source_resolution if self.source_resolution else '', - 'height': '720' if self.source_resolution else '', - 'width': '1280' if self.source_resolution else '', - 'vcodec': self.source_vcodec.lower() if self.source_vcodec else '', - 'acodec': self.source_acodec.lower(), - 'fps': '24' if self.source_resolution else '', - 'hdr': 'hdr' if self.source_resolution else '' - } - - def get_example_media_format(self): - try: - return self.media_format.format(**self.example_media_format_dict) - except Exception as e: - return '' - - def is_regex_match(self, media_item_title): - if not self.filter_text: - return True - return bool(re.search(self.filter_text, media_item_title)) - - def get_index(self, type): - indexer = self.INDEXERS.get(self.source_type, None) - if not callable(indexer): - raise Exception(f'Source type f"{self.source_type}" has no indexer') - days = None - if self.download_cap_date: - days = timedelta(seconds=self.download_cap).days - response = indexer(self.get_index_url(type=type), days=days) - if not isinstance(response, dict): - return [] - entries = response.get('entries', []) - return entries - - def index_media(self): - ''' - Index the media source returning a list of media metadata as dicts. - ''' - entries = list() - if self.index_videos: - entries += self.get_index('videos') - # Playlists do something different that I have yet to figure out - if not self.is_playlist: - if self.index_streams: - entries += self.get_index('streams') - - if settings.MAX_ENTRIES_PROCESSING: - entries = entries[:settings.MAX_ENTRIES_PROCESSING] - return entries - def get_media_thumb_path(instance, filename): # we don't want to use alternate names for thumb files if instance.thumb: From babc0345d6f9d31b2d27a95719501f62e73e0863 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:34:49 -0400 Subject: [PATCH 108/174] Update and rename legacy.py to media.py --- tubesync/sync/models/{legacy.py => media.py} | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) rename tubesync/sync/models/{legacy.py => media.py} (99%) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/media.py similarity index 99% rename from tubesync/sync/models/legacy.py rename to tubesync/sync/models/media.py index 83bd9256..418e626d 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/media.py @@ -36,9 +36,8 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) - -media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') -_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) +from .source import Source +from .misc import media_file_storage, _srctype_dict def get_media_thumb_path(instance, filename): # we don't want to use alternate names for thumb files From e8777f2ecb4fd6c44d25b5501ed87a88559723f3 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:40:19 -0400 Subject: [PATCH 109/174] Create misc.py --- tubesync/sync/models/misc.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tubesync/sync/models/misc.py diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py new file mode 100644 index 00000000..383e0bba --- /dev/null +++ b/tubesync/sync/models/misc.py @@ -0,0 +1,8 @@ +from django.conf import settings +from django.core.files.storage import FileSystemStorage +from ..choices import Val, YouTube_SourceType + + +media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') +_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + From 37fcabb242976969846feb95c46e9316ff3c4816 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:42:20 -0400 Subject: [PATCH 110/174] Update __init__.py --- tubesync/sync/models/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index ff0f7b3b..f443c7c3 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,6 +1,5 @@ +from .media import Media from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer - -from .legacy import * From 6d288ec4477b906bdd46dd2f59c5e894b337b0cb Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:43:03 -0400 Subject: [PATCH 111/174] Update source.py --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 01a0d36e..ce0bf5ab 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -19,7 +19,7 @@ from ..youtube import ( get_media_info as get_youtube_media_info, get_channel_image_info as get_youtube_channel_image_info, ) -from .legacy import _srctype_dict, media_file_storage +from .misc import _srctype_dict, media_file_storage class Source(db.models.Model): From 7cad4ce396339d1587b0c4de3a6b99d3e587a595 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:45:42 -0400 Subject: [PATCH 112/174] Update source.py --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index ce0bf5ab..0dcfbc98 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -10,7 +10,7 @@ from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from ..choices import (Val, SponsorBlock_Category, YouTube_SourceType, IndexSchedule, - CapChoices, FilterSeconds, FileExtension, + CapChoices, Fallback, FileExtension, FilterSeconds, SourceResolution, SourceResolutionInteger, YouTube_VideoCodec, YouTube_AudioCodec, ) From 5b79293562dbdc3dfb9346a862700d54d3e24217 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:47:58 -0400 Subject: [PATCH 113/174] Update source.py --- tubesync/sync/models/source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 0dcfbc98..7ab9ea4b 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -5,6 +5,7 @@ from pathlib import Path from django import db from django.conf import settings from django.core.exceptions import SuspiciousOperation +from django.core.validators import RegexValidator from django.utils import timezone from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ From 109608c9af94c33339e85cc5965cbd25b097172d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:48:48 -0400 Subject: [PATCH 114/174] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index e062ea0c..17d214fb 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,7 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -from .legacy import Media +from .media import Media class Metadata(db.models.Model): From 4cd2178edb55b6f2bb63679b24f816180a37cc40 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:51:30 -0400 Subject: [PATCH 115/174] Update __init__.py --- tubesync/sync/models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index f443c7c3..a9324f0e 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -3,3 +3,5 @@ from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer + +from .misc import media_file_storage From 004d98efa54e5f5149036a3647bec0a731f86d6b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:58:28 -0400 Subject: [PATCH 116/174] Update media.py --- tubesync/sync/models/media.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 418e626d..7f179f13 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -37,20 +37,10 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) from .source import Source -from .misc import media_file_storage, _srctype_dict - -def get_media_thumb_path(instance, filename): - # we don't want to use alternate names for thumb files - if instance.thumb: - instance.thumb.delete(save=False) - fileid = str(instance.uuid).lower() - filename = f'{fileid}.jpg' - prefix = fileid[:2] - return Path('thumbs') / prefix / filename - - -def get_media_file_path(instance, filename): - return instance.filepath +from .misc import ( + media_file_storage, _srctype_dict, + get_media_thumb_path, get_media_file_path, +) class Media(models.Model): From 951b24bbefddd8d31b8c90e90ce99b56f8a4da98 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:01:49 -0400 Subject: [PATCH 117/174] Update misc.py --- tubesync/sync/models/misc.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py index 383e0bba..159e0fa3 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/misc.py @@ -1,3 +1,4 @@ +from pathlib import Path from django.conf import settings from django.core.files.storage import FileSystemStorage from ..choices import Val, YouTube_SourceType @@ -6,3 +7,17 @@ from ..choices import Val, YouTube_SourceType media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + +def get_media_file_path(instance, filename): + return instance.filepath + + +def get_media_thumb_path(instance, filename): + # we don't want to use alternate names for thumb files + if instance.thumb: + instance.thumb.delete(save=False) + fileid = str(instance.uuid).lower() + filename = f'{fileid}.jpg' + prefix = fileid[:2] + return Path('thumbs') / prefix / filename + From 35697177b78651d357d10551526ce4e2c31df37d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:03:05 -0400 Subject: [PATCH 118/174] Update __init__.py --- tubesync/sync/models/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index a9324f0e..e28c7db7 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -4,4 +4,8 @@ from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer -from .misc import media_file_storage +from .misc import ( + get_media_file_path, + get_media_thumb_path, + media_file_storage, +) From 971be3d3dd7c584cb1d03b32adddc287dee604d9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:05:53 -0400 Subject: [PATCH 119/174] Update __init__.py --- tubesync/sync/models/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index e28c7db7..124e1ad9 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,3 +1,5 @@ +from common.json import JSONEncoder + from .media import Media from .source import Source from .metadata import Metadata @@ -5,7 +7,7 @@ from .metadata_format import MetadataFormat from .media_server import MediaServer from .misc import ( - get_media_file_path, + #get_media_file_path, get_media_thumb_path, media_file_storage, ) From 46e6030d114b4798884b5f9a42c44379631128ec Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:07:21 -0400 Subject: [PATCH 120/174] Update __init__.py --- tubesync/sync/models/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 124e1ad9..cd7861d3 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -7,7 +7,7 @@ from .metadata_format import MetadataFormat from .media_server import MediaServer from .misc import ( - #get_media_file_path, - get_media_thumb_path, + get_media_file_path, + #get_media_thumb_path, media_file_storage, ) From f9d9e7b66bcc750f70933e0be543ac69a08d7fab Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:08:47 -0400 Subject: [PATCH 121/174] Update __init__.py --- tubesync/sync/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index cd7861d3..f2649d4d 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -8,6 +8,6 @@ from .media_server import MediaServer from .misc import ( get_media_file_path, - #get_media_thumb_path, + get_media_thumb_path, media_file_storage, ) From 080acb7995840b5b1030d36f6fdf5a865222b1d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:10:52 -0400 Subject: [PATCH 122/174] Update __init__.py --- tubesync/sync/models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index f2649d4d..e951340d 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,5 +1,7 @@ from common.json import JSONEncoder +from ..fields import CommaSepChoiceField + from .media import Media from .source import Source from .metadata import Metadata From 0a59c1dd97898813d820d6846dafc4db2a749cb1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:24:14 -0400 Subject: [PATCH 123/174] Update media.py --- tubesync/sync/models/media.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 7f179f13..6be36b83 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1,7 +1,6 @@ import os import uuid import json -import re from collections import OrderedDict from copy import deepcopy from datetime import datetime, timedelta, timezone as tz @@ -9,16 +8,12 @@ from pathlib import Path from xml.etree import ElementTree from django.conf import settings from django.db import models -from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation -from django.core.files.storage import FileSystemStorage -from django.core.serializers.json import DjangoJSONEncoder -from django.core.validators import RegexValidator +from django.core.exceptions import ObjectDoesNotExist from django.db.transaction import atomic from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log -from common.json import JSONEncoder from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) @@ -30,7 +25,6 @@ from ..utils import (seconds_to_timestr, parse_media_format, filter_response, multi_key_sort) from ..matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) -from ..fields import CommaSepChoiceField from ..choices import ( Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, MediaState, SourceResolution, SourceResolutionInteger, From 1a3a86cde422840aeedfb8da1f16dc8c90c5902a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:37:04 -0400 Subject: [PATCH 124/174] Update settings.py --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 9960f60e..391a209d 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.0' +VERSION = '0.15.1' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From 6278087b12ef1f29f4c440a30db6be3da751f42a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 03:04:12 -0400 Subject: [PATCH 125/174] Update media.py --- tubesync/sync/models/media.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 6be36b83..ce351267 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,19 +17,15 @@ from common.logger import log from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) -from ..youtube import ( get_media_info as get_youtube_media_info, - download_media as download_youtube_media, - get_channel_image_info as get_youtube_channel_image_info) +from ..youtube import ( get_media_info as get_youtube_media_info, + download_media as download_youtube_media) from ..utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote, multi_key_sort) -from ..matching import ( get_best_combined_format, get_best_audio_format, +from ..matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) -from ..choices import ( Val, CapChoices, Fallback, FileExtension, - FilterSeconds, IndexSchedule, MediaServerType, - MediaState, SourceResolution, SourceResolutionInteger, - SponsorBlock_Category, YouTube_AudioCodec, - YouTube_SourceType, YouTube_VideoCodec) +from ..choices import ( Val, Fallback, MediaState, SourceResolution, + YouTube_AudioCodec, YouTube_VideoCodec) from .source import Source from .misc import ( media_file_storage, _srctype_dict, From 3bb19861cd9295dbb7bca7cd0748d4c8a22a9f86 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 03:40:20 -0400 Subject: [PATCH 126/174] Add `_nfo_element` function --- tubesync/sync/models/misc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py index 159e0fa3..81257966 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/misc.py @@ -8,6 +8,13 @@ media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), bas _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) +def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): + element = nfo.makeelement(label, attrs) + element.text = text + element.tail = tail + (char * indent) + return element + + def get_media_file_path(instance, filename): return instance.filepath From 189df4d72ae2f4600970c64facd329be8d2615df Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 04:17:48 -0400 Subject: [PATCH 127/174] Use `_nfo_element` function --- tubesync/sync/models/media.py | 116 ++++++++++++++-------------------- 1 file changed, 48 insertions(+), 68 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index ce351267..eed6103a 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -28,7 +28,7 @@ from ..choices import ( Val, Fallback, MediaState, SourceResolution, YouTube_AudioCodec, YouTube_VideoCodec) from .source import Source from .misc import ( - media_file_storage, _srctype_dict, + media_file_storage, _srctype_dict, _nfo_element, get_media_thumb_path, get_media_file_path, ) @@ -942,37 +942,27 @@ class Media(models.Model): nfo = ElementTree.Element('episodedetails') nfo.text = '\n ' # title = media metadata title - title = nfo.makeelement('title', {}) - title.text = clean_emoji(self.title) - title.tail = '\n ' - nfo.append(title) + nfo.append(_nfo_element(nfo, + 'title', clean_emoji(self.title), + )) # showtitle = source name - showtitle = nfo.makeelement('showtitle', {}) - showtitle.text = clean_emoji(str(self.source.name).strip()) - showtitle.tail = '\n ' - nfo.append(showtitle) + nfo.append(_nfo_element(nfo, + 'showtitle', clean_emoji(str(self.source.name).strip()), + )) # season = upload date year - season = nfo.makeelement('season', {}) - if self.source.is_playlist: - # If it's a playlist, set season to 1 - season.text = '1' - else: - # If it's not a playlist, set season to upload date year - season.text = str(self.upload_date.year) if self.upload_date else '' - season.tail = '\n ' - nfo.append(season) + nfo.append(_nfo_element(nfo, + 'season', + '1' if self.source.is_playlist else str( + self.upload_date.year if self.upload_date else '' + ), + )) # episode = number of video in the year - episode = nfo.makeelement('episode', {}) - episode.text = self.get_episode_str() - episode.tail = '\n ' - nfo.append(episode) + nfo.append(_nfo_element(nfo, + 'episode', self.get_episode_str(), + )) # ratings = media metadata youtube rating - value = nfo.makeelement('value', {}) - value.text = str(self.rating) - value.tail = '\n ' - votes = nfo.makeelement('votes', {}) - votes.text = str(self.votes) - votes.tail = '\n ' + value = _nfo_element(nfo, 'value', str(self.rating), indent=6) + votes = _nfo_element(nfo, 'votes', str(self.votes), indent=4) rating_attrs = OrderedDict() rating_attrs['name'] = 'youtube' rating_attrs['max'] = '5' @@ -989,61 +979,51 @@ class Media(models.Model): ratings.tail = '\n ' nfo.append(ratings) # plot = media metadata description - plot = nfo.makeelement('plot', {}) - plot.text = clean_emoji(str(self.description).strip()) - plot.tail = '\n ' - nfo.append(plot) + nfo.append(_nfo_element(nfo, + 'plot', clean_emoji(str(self.description).strip()), + )) # thumb = local path to media thumbnail - thumb = nfo.makeelement('thumb', {}) - thumb.text = self.thumbname if self.source.copy_thumbnails else '' - thumb.tail = '\n ' - nfo.append(thumb) + nfo.append(_nfo_element(nfo, + 'thumb', self.thumbname if self.source.copy_thumbnails else '', + )) # mpaa = media metadata age requirement - mpaa = nfo.makeelement('mpaa', {}) - mpaa.text = str(self.age_limit) - mpaa.tail = '\n ' if self.age_limit and self.age_limit > 0: - nfo.append(mpaa) + nfo.append(_nfo_element(nfo, + 'mpaa', str(self.age_limit), + )) # runtime = media metadata duration in seconds - runtime = nfo.makeelement('runtime', {}) - runtime.text = str(self.duration) - runtime.tail = '\n ' - nfo.append(runtime) + nfo.append(_nfo_element(nfo, + 'runtime', str(self.duration), + )) # id = media key - idn = nfo.makeelement('id', {}) - idn.text = str(self.key).strip() - idn.tail = '\n ' - nfo.append(idn) + nfo.append(_nfo_element(nfo, + 'id', str(self.key).strip(), + )) # uniqueid = media key uniqueid_attrs = OrderedDict() uniqueid_attrs['type'] = 'youtube' uniqueid_attrs['default'] = 'True' - uniqueid = nfo.makeelement('uniqueid', uniqueid_attrs) - uniqueid.text = str(self.key).strip() - uniqueid.tail = '\n ' - nfo.append(uniqueid) + nfo.append(_nfo_element(nfo, + 'uniqueid', str(self.key).strip(), attrs=uniqueid_attrs, + )) # studio = media metadata uploader - studio = nfo.makeelement('studio', {}) - studio.text = clean_emoji(str(self.uploader).strip()) - studio.tail = '\n ' - nfo.append(studio) + nfo.append(_nfo_element(nfo, + 'studio', clean_emoji(str(self.uploader).strip()), + )) # aired = media metadata uploaded date - aired = nfo.makeelement('aired', {}) upload_date = self.upload_date - aired.text = upload_date.strftime('%Y-%m-%d') if upload_date else '' - aired.tail = '\n ' - nfo.append(aired) + nfo.append(_nfo_element(nfo, + 'aired', upload_date.strftime('%Y-%m-%d') if upload_date else '', + )) # dateadded = date and time media was created in tubesync - dateadded = nfo.makeelement('dateadded', {}) - dateadded.text = self.created.strftime('%Y-%m-%d %H:%M:%S') - dateadded.tail = '\n ' - nfo.append(dateadded) + nfo.append(_nfo_element(nfo, + 'dateadded', self.created.strftime('%Y-%m-%d %H:%M:%S'), + )) # genre = any media metadata categories if they exist for category_str in self.categories: - genre = nfo.makeelement('genre', {}) - genre.text = str(category_str).strip() - genre.tail = '\n ' - nfo.append(genre) + nfo.append(_nfo_element(nfo, + 'genre', str(category_str).strip(), + )) nfo[-1].tail = '\n' # Return XML tree as a prettified string return ElementTree.tostring(nfo, encoding='utf8', method='xml').decode('utf8') From 9aa5e6b9675f4b0d9c501842baa010237927814b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:54:43 -0400 Subject: [PATCH 128/174] Create a media entry after deletion --- tubesync/sync/signals.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 32b0b5f6..8218c438 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,6 +295,25 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=instance.get_metadata_first_value( + 'thumbnail', + arg_dict=existing_metadata, + ), + ), + ) + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -370,3 +389,32 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) + # Create a media entry for the indexing task to find + # Requirements: + # source, key, duration, title, published + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) + if created: + old_metadata = instance.loaded_metadata + skipped_media.downloaded = False + skipped_media.duration = instance.duration + skipped_media.metadata = skipped_media.metadata_dumps( + arg_dict=dict( + _media_instance_was_deleted=True, + site=old_metadata.get('site'), + thumbnail=old_metadata.get('thumbnail'), + ), + ) + skipped_media.published = instance.published + skipped_media.title = instance.title + skipped_media.skip = True + skipped_media.manual_skip = True + skipped_media.save() + Metadata.objects.filter( + media__isnull=True, + site=old_metadata.get('site') or 'Youtube', + key=skipped_media.key, + ).update(media=skipped_media) + From f86b666c125446a116c1fddae5ecc90463bc3f52 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:57:23 -0400 Subject: [PATCH 129/174] fixup: import `Metadata` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 8218c438..49b0145d 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from .models import Source, Media, MediaServer +from .models import Source, Media, MediaServer, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, From d1fd568066a27c5804a097b34d47e57de3a141d6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:11:18 -0400 Subject: [PATCH 130/174] Test for the specific media items, not any task --- tubesync/sync/tests.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 303aa18a..b5440291 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,8 +469,19 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} - download_media_thumbnail_tasks = Task.objects.filter(**q) - self.assertFalse(download_media_thumbnail_tasks) + found_thumbnail_task1 = False + found_thumbnail_task2 = False + found_thumbnail_task3 = False + for task in Task.objects.filter(**q): + if test_media1_pk in task.task_params: + found_thumbnail_task1 = True + if test_media2_pk in task.task_params: + found_thumbnail_task2 = True + if test_media3_pk in task.task_params: + found_thumbnail_task3 = True + self.assertFalse(found_thumbnail_task1) + self.assertFalse(found_thumbnail_task2) + self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 4a04326d16737f9bbc7511ddc3c6c0e8f2129c92 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:31:49 -0400 Subject: [PATCH 131/174] Change the expected count and verify the key matches --- tubesync/sync/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b5440291..b84ffe03 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1847,5 +1847,6 @@ class TasksTestCase(TestCase): cleanup_old_media() self.assertEqual(src1.media_source.all().count(), 3) - self.assertEqual(src2.media_source.all().count(), 2) + self.assertEqual(src2.media_source.all().count(), 3) self.assertEqual(Media.objects.filter(pk=m22.pk).exists(), False) + self.assertEqual(Media.objects.filter(source=src2, key=m22.key, skip=True).exists(), True) From ba451d9401a0834a80b6d92f578e88497b704352 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:34:53 -0400 Subject: [PATCH 132/174] testing: sanity check --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 49b0145d..877cc4ed 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -396,7 +396,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if created: + if False and created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From a80623bce40e51a1755e4af448fb944d331cf964 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:41:14 -0400 Subject: [PATCH 133/174] testing: no save in pre-delete --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 877cc4ed..efb273a0 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,7 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - instance.save() + # TODO: instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) From 5a9c3050c8069d07092964af27a7682970b5dcfd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:50:37 -0400 Subject: [PATCH 134/174] Fixes from CI tests --- tubesync/sync/signals.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index efb273a0..26b18ecc 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,9 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - # TODO: instance.save() + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -396,7 +398,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if False and created: + if created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From 9d8a5574e416b0a9881e4777c44d8325e46ccb29 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:01:15 -0400 Subject: [PATCH 135/174] Delete tasks first then modify metadata --- tubesync/sync/signals.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 26b18ecc..76731598 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,27 +295,6 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): - # Save the metadata site & thumbnail URL to the metadata column - existing_metadata = instance.loaded_metadata - metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( - deleted=True, - site=instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail=instance.get_metadata_first_value( - 'thumbnail', - arg_dict=existing_metadata, - ), - ), - ) - # Do not create more tasks before deleting - instance.manual_skip = True - instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -331,6 +310,24 @@ def media_pre_delete(sender, instance, **kwargs): # Remove thumbnail file for deleted media if instance.thumb: instance.thumb.delete(save=False) + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=thumbnail_url, + ), + ) + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() @receiver(post_delete, sender=Media) From f7ca8189a0e61cf8e3f6b1f6c065c2f45d7fa5f1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:04:27 -0400 Subject: [PATCH 136/174] Update tests.py --- tubesync/sync/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b84ffe03..2926a296 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,10 +469,12 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} + download_media_thumbnail_tasks = Task.objects.filter(**q) + self.assertFalse(download_media_thumbnail_tasks) found_thumbnail_task1 = False found_thumbnail_task2 = False found_thumbnail_task3 = False - for task in Task.objects.filter(**q): + for task in download_media_thumbnail_tasks: if test_media1_pk in task.task_params: found_thumbnail_task1 = True if test_media2_pk in task.task_params: From 147f245e97d1cabd31955496679fedc87324ad1f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:06:28 -0400 Subject: [PATCH 137/174] Update tests.py --- tubesync/sync/tests.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 2926a296..24f0d092 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -471,19 +471,6 @@ class FrontEndTestCase(TestCase): q = {'task_name': 'sync.tasks.download_media_thumbnail'} download_media_thumbnail_tasks = Task.objects.filter(**q) self.assertFalse(download_media_thumbnail_tasks) - found_thumbnail_task1 = False - found_thumbnail_task2 = False - found_thumbnail_task3 = False - for task in download_media_thumbnail_tasks: - if test_media1_pk in task.task_params: - found_thumbnail_task1 = True - if test_media2_pk in task.task_params: - found_thumbnail_task2 = True - if test_media3_pk in task.task_params: - found_thumbnail_task3 = True - self.assertFalse(found_thumbnail_task1) - self.assertFalse(found_thumbnail_task2) - self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 7a8421e11acf0921eee516d13e7395934b6b1f01 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:10:14 -0400 Subject: [PATCH 138/174] Use the field mapping for site & thumbnail --- tubesync/sync/signals.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 76731598..9bec1401 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -314,16 +314,19 @@ def media_pre_delete(sender, instance, **kwargs): existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' column_metadata = instance.metadata_loads(metadata_str) + site_field = instance.get_metadata_field('extractor_key') + thumbnail_field = instance.get_metadata_field('thumbnail') instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( + arg_dict=dict(column_metadata).update(dict( deleted=True, - site=instance.get_metadata_first_value( + ).update({ + site_field: instance.get_metadata_first_value( 'extractor_key', 'Youtube', arg_dict=existing_metadata, ), - thumbnail=thumbnail_url, - ), + thumbnail_field: thumbnail_url, + })), ) # Do not create more tasks before deleting instance.manual_skip = True @@ -396,15 +399,19 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + site_field = instance.get_metadata_field('extractor_key') + thumbnail_url = instance.thumbnail + thumbnail_field = instance.get_metadata_field('thumbnail') old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration skipped_media.metadata = skipped_media.metadata_dumps( arg_dict=dict( _media_instance_was_deleted=True, - site=old_metadata.get('site'), - thumbnail=old_metadata.get('thumbnail'), - ), + ).update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }), ) skipped_media.published = instance.published skipped_media.title = instance.title @@ -413,7 +420,7 @@ def media_post_delete(sender, instance, **kwargs): skipped_media.save() Metadata.objects.filter( media__isnull=True, - site=old_metadata.get('site') or 'Youtube', + site=old_metadata.get(site_field) or 'Youtube', key=skipped_media.key, ).update(media=skipped_media) From 691d049e2cd74f36882c4bf9ae759f9f0d367e3b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:39:57 -0400 Subject: [PATCH 139/174] fixup: update `dict` first --- tubesync/sync/signals.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 9bec1401..1743914a 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,21 +313,18 @@ def media_pre_delete(sender, instance, **kwargs): # Save the metadata site & thumbnail URL to the metadata column existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) + arg_dict = instance.metadata_loads(metadata_str) site_field = instance.get_metadata_field('extractor_key') thumbnail_field = instance.get_metadata_field('thumbnail') - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update(dict( - deleted=True, - ).update({ - site_field: instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail_field: thumbnail_url, - })), - ) + arg_dict.update({ + site_field: instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail_field: thumbnail_url, + }) + instance.metadata = instance.metadata_dumps(arg_dict=arg_dict) # Do not create more tasks before deleting instance.manual_skip = True instance.save() @@ -399,19 +396,21 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key') thumbnail_url = instance.thumbnail thumbnail_field = instance.get_metadata_field('thumbnail') - old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration + arg_dict=dict( + _media_instance_was_deleted=True, + ) + arg_dict.update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }) skipped_media.metadata = skipped_media.metadata_dumps( - arg_dict=dict( - _media_instance_was_deleted=True, - ).update({ - site_field: old_metadata.get(site_field), - thumbnail_field: thumbnail_url, - }), + arg_dict=arg_dict, ) skipped_media.published = instance.published skipped_media.title = instance.title From 7beda36d87523c802c1b076925c85f6813e1d5a2 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:02:03 -0400 Subject: [PATCH 140/174] Mark media for deletion --- tubesync/sync/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2e6b1433..d29e8239 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -933,6 +933,10 @@ def delete_all_media_for_source(source_id, source_name, source_directory): for media in qs_gen(mqs): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): + #media.downloaded = False + media.skip = True + media.manual_skip = True + media.save() media.delete() # Remove the directory, if the user requested that directory_path = Path(source_directory) From 071508e2df128cae07ff3d0e922530627da7a4cd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:20:57 -0400 Subject: [PATCH 141/174] Coordination with deletion of `Source` instances --- tubesync/sync/signals.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 1743914a..6a2dd22f 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -333,7 +333,13 @@ def media_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): # Remove the video file, when configured to do so - if instance.source.delete_files_on_disk and instance.media_file: + remove_files = ( + instance.source and + instance.source.delete_files_on_disk and + instance.downloaded and + instance.media_file + ) + if remove_files: video_path = Path(str(instance.media_file.path)).resolve(strict=False) instance.media_file.delete(save=False) # the other files we created have these known suffixes @@ -391,10 +397,19 @@ def media_post_delete(sender, instance, **kwargs): # Create a media entry for the indexing task to find # Requirements: # source, key, duration, title, published - skipped_media, created = Media.objects.get_or_create( - key=instance.key, - source=instance.source, + created = False + create_for_indexing_task = ( + not ( + #not instance.downloaded and + instance.skip and + instance.manual_skip + ) ) + if create_for_indexing_task: + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) if created: old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key') From 2a075720e1e51c94ba4da135336af8ca8dbef268 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 01:29:29 -0400 Subject: [PATCH 142/174] Re-order imports --- tubesync/sync/models/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index e951340d..362095e7 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -2,14 +2,15 @@ from common.json import JSONEncoder from ..fields import CommaSepChoiceField +from .misc import ( + get_media_file_path, + get_media_thumb_path, + media_file_storage, +) + from .media import Media from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer -from .misc import ( - get_media_file_path, - get_media_thumb_path, - media_file_storage, -) From e697fc70a11a39622dd5c6e7abc1c4ccb00f0a21 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 01:36:16 -0400 Subject: [PATCH 143/174] Use `JSONEncoder` for `MediaServer` --- tubesync/sync/models/media_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index a8ecefc6..74502fac 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1,3 +1,4 @@ +from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ from ..choices import Val, MediaServerType @@ -45,6 +46,7 @@ class MediaServer(db.models.Model): ) options = db.models.JSONField( _('options'), + encoder=JSONEncoder, blank=False, null=True, help_text=_('Options for the media server'), From 25f5c1cf5b2c9ea4a81de254411b80b0fae7855e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 02:26:02 -0400 Subject: [PATCH 144/174] Add migration file via upload --- ...ons_alter_source_source_acodec_and_more.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py diff --git a/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py b/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py new file mode 100644 index 00000000..46ea113f --- /dev/null +++ b/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.1.9 on 2025-05-10 06:18 + +import common.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0032_metadata_transfer'), + ] + + operations = [ + migrations.AlterField( + model_name='mediaserver', + name='options', + field=models.JSONField(encoder=common.json.JSONEncoder, help_text='Options for the media server', null=True, verbose_name='options'), + ), + migrations.AlterField( + model_name='source', + name='source_acodec', + field=models.CharField(choices=[('OPUS', 'OPUS'), ('MP4A', 'MP4A')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'), + ), + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AV1', 'AV1'), ('VP9', 'VP9'), ('AVC1', 'AVC1 (H.264)')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + ] From a927c0f6de8d91a56594f2efb2397e16d139973e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:08:58 -0400 Subject: [PATCH 145/174] Add comments about the imports --- tubesync/sync/models/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 362095e7..41028293 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,13 +1,21 @@ +# These are referenced from the migration files +# TODO: update migration files to remove +# CommaSepChoiceField and JSONEncoder + from common.json import JSONEncoder from ..fields import CommaSepChoiceField +# Used by migration files and staying here + from .misc import ( get_media_file_path, get_media_thumb_path, media_file_storage, ) +# The actual model classes + from .media import Media from .source import Source from .metadata import Metadata From 58c20d669a4c85cf91e07aac1d37e0a960ab32fc Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:17:15 -0400 Subject: [PATCH 146/174] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 00efa0f6..aee89518 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -1,7 +1,7 @@ # Generated by Django 5.1.8 on 2025-04-11 07:36 +import common.json import django.db.models.deletion -import sync.models import uuid from django.db import migrations, models @@ -23,7 +23,7 @@ class Migration(migrations.Migration): ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), ], options={ @@ -40,7 +40,7 @@ class Migration(migrations.Migration): ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), ], options={ From 21974612999a0a7eabc4f706e206bb1a8664098d Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:20:13 -0400 Subject: [PATCH 147/174] Update 0031_squashed_metadata_metadataformat.py --- .../migrations/0031_squashed_metadata_metadataformat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index 13189f10..c7a78bd8 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -1,7 +1,7 @@ # Generated by Django 5.1.8 on 2025-04-23 18:10 +import common.json import django.db.models.deletion -import sync.models import uuid from django.db import migrations, models @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), ('uploaded', models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), ('media', models.OneToOneField(help_text='Media the metadata belongs to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='new_metadata', to='sync.media')), ], options={ @@ -43,7 +43,7 @@ class Migration(migrations.Migration): ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata')), ], options={ From 24d126ad7d47e4931eba89800d77c4d8ff53bedd Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:22:05 -0400 Subject: [PATCH 148/174] Update __init__.py --- tubesync/sync/models/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 41028293..3e4d521b 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,8 +1,6 @@ # These are referenced from the migration files # TODO: update migration files to remove -# CommaSepChoiceField and JSONEncoder - -from common.json import JSONEncoder +# CommaSepChoiceField from ..fields import CommaSepChoiceField From e1faca4c3d8af84d205326179aad5d5fa0eba953 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:29:59 -0400 Subject: [PATCH 149/174] Update 0016_auto_20230214_2052.py --- tubesync/sync/migrations/0016_auto_20230214_2052.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0016_auto_20230214_2052.py b/tubesync/sync/migrations/0016_auto_20230214_2052.py index ffba1952..d4319759 100644 --- a/tubesync/sync/migrations/0016_auto_20230214_2052.py +++ b/tubesync/sync/migrations/0016_auto_20230214_2052.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.18 on 2023-02-14 20:52 from django.db import migrations, models -import sync.models +import sync.fields class Migration(migrations.Migration): @@ -29,6 +29,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='source', name='sponsorblock_categories', - field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), + field=sync.fields.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), ), ] From 26372efe25b52aaf666d60c466c19c4b98d73bf6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:31:52 -0400 Subject: [PATCH 150/174] Update __init__.py --- tubesync/sync/models/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 3e4d521b..dba06dfb 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,10 +1,4 @@ # These are referenced from the migration files -# TODO: update migration files to remove -# CommaSepChoiceField - -from ..fields import CommaSepChoiceField - -# Used by migration files and staying here from .misc import ( get_media_file_path, From 0d1f066fced3d1063b9fe56e2862188b4396a276 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 04:21:06 -0400 Subject: [PATCH 151/174] Reorganize misc.py --- tubesync/sync/models/__init__.py | 13 ++++++++----- tubesync/sync/models/{misc.py => _migrations.py} | 9 --------- tubesync/sync/models/_private.py | 12 ++++++++++++ tubesync/sync/models/media.py | 8 ++++---- tubesync/sync/models/source.py | 3 ++- 5 files changed, 26 insertions(+), 19 deletions(-) rename tubesync/sync/models/{misc.py => _migrations.py} (62%) create mode 100644 tubesync/sync/models/_private.py diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index dba06dfb..d7ed077c 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,16 +1,19 @@ # These are referenced from the migration files -from .misc import ( +from ._migrations import ( get_media_file_path, get_media_thumb_path, media_file_storage, ) # The actual model classes +# The order starts with independent classes +# then the classes that depend on them follow. -from .media import Media -from .source import Source -from .metadata import Metadata -from .metadata_format import MetadataFormat from .media_server import MediaServer +from .source import Source +from .media import Media +from .metadata import Metadata +from .metadata_format import MetadataFormat + diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/_migrations.py similarity index 62% rename from tubesync/sync/models/misc.py rename to tubesync/sync/models/_migrations.py index 81257966..5ca5d101 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/_migrations.py @@ -1,18 +1,9 @@ from pathlib import Path from django.conf import settings from django.core.files.storage import FileSystemStorage -from ..choices import Val, YouTube_SourceType media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') -_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) - - -def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): - element = nfo.makeelement(label, attrs) - element.text = text - element.tail = tail + (char * indent) - return element def get_media_file_path(instance, filename): diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py new file mode 100644 index 00000000..96539dbe --- /dev/null +++ b/tubesync/sync/models/_private.py @@ -0,0 +1,12 @@ +from ..choices import Val, YouTube_SourceType + + +_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + + +def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): + element = nfo.makeelement(label, attrs) + element.text = text + element.tail = tail + (char * indent) + return element + diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index eed6103a..92e2b23b 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -26,11 +26,11 @@ from ..matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from ..choices import ( Val, Fallback, MediaState, SourceResolution, YouTube_AudioCodec, YouTube_VideoCodec) -from .source import Source -from .misc import ( - media_file_storage, _srctype_dict, _nfo_element, - get_media_thumb_path, get_media_file_path, +from ._migrations import ( + media_file_storage, get_media_thumb_path, get_media_file_path, ) +from ._private import _srctype_dict, _nfo_element +from .source import Source class Media(models.Model): diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 7ab9ea4b..74f75278 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -20,7 +20,8 @@ from ..youtube import ( get_media_info as get_youtube_media_info, get_channel_image_info as get_youtube_channel_image_info, ) -from .misc import _srctype_dict, media_file_storage +from ._migrations import media_file_storage +from ._private import _srctype_dict class Source(db.models.Model): From 5c483d810c1afd7a850ce9e54fcb1533693d9618 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 04:31:14 -0400 Subject: [PATCH 152/174] Re-indent imports --- tubesync/sync/models/media.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 92e2b23b..daaf723d 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -15,17 +15,27 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException -from common.utils import ( clean_filename, clean_emoji, - django_queryset_generator as qs_gen, ) -from ..youtube import ( get_media_info as get_youtube_media_info, - download_media as download_youtube_media) -from ..utils import (seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote, - multi_key_sort) -from ..matching import (get_best_combined_format, get_best_audio_format, - get_best_video_format) -from ..choices import ( Val, Fallback, MediaState, SourceResolution, - YouTube_AudioCodec, YouTube_VideoCodec) +from common.utils import ( + clean_filename, clean_emoji, + django_queryset_generator as qs_gen, +) +from ..youtube import ( + get_media_info as get_youtube_media_info, + download_media as download_youtube_media, +) +from ..utils import ( + seconds_to_timestr, parse_media_format, filter_response, + write_text_file, mkdir_p, directory_and_stem, glob_quote, + multi_key_sort, +) +from ..matching import ( + get_best_combined_format, + get_best_audio_format, get_best_video_format, +) +from ..choices import ( + Val, Fallback, MediaState, SourceResolution, + YouTube_AudioCodec, YouTube_VideoCodec, +) from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) From 12cdcdec96b48284f1a957b6ff11ec98b6744b31 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 05:21:26 -0400 Subject: [PATCH 153/174] Use a widget to select `when` in the browser --- tubesync/sync/forms.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index c0bfd13f..6df609da 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -49,6 +49,13 @@ class ScheduleTaskForm(forms.Form): when = forms.DateTimeField( label=_('When the task should run'), required=True, + #widget=forms.SplitDateTimeWidget( + # date_attrs={'type': 'date'}, + # time_attrs={'type': 'time'}, + #), + widget=forms.DateTimeInput( + attrs={'type': 'datetime-local'}, + ), ) From 3f5bd0746eb8bcdf21a1202e1953a8e7d52cba70 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 05:57:01 -0400 Subject: [PATCH 154/174] Adjust the description and button --- tubesync/sync/templates/sync/task-schedule.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/templates/sync/task-schedule.html b/tubesync/sync/templates/sync/task-schedule.html index 8f387b4a..63af2eb3 100644 --- a/tubesync/sync/templates/sync/task-schedule.html +++ b/tubesync/sync/templates/sync/task-schedule.html @@ -15,8 +15,8 @@ other tasks are taking to complete the assigned work.

- This will change the time that the task is requesting to be run - to the current time, plus some number of seconds. + This will change the time that the task is requesting to be the + current time, or a chosen future time.

@@ -26,7 +26,7 @@ {% include 'simpleform.html' with form=form %}
- +
From 05eb5cdeb0677805a848ee064326d03e8f2c5a4c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 06:48:46 -0400 Subject: [PATCH 155/174] Handle the timestamp from the URL correctly --- tubesync/sync/views.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 692675a0..dcf8e3e7 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1026,18 +1026,16 @@ class TaskScheduleView(FormView, SingleObjectMixin): def dispatch(self, request, *args, **kwargs): self.object = self.get_object() - self.timestamp = kwargs.get('timestamp', '') + self.timestamp = kwargs.get('timestamp') try: - self.timestamp = int(self.timestamp, 10) - except (TypeError, ValueError): - self.timestamp = None - else: - try: - self.when = timestamp_to_datetime(self.timestamp) - except AssertionError: - self.when = None + self.when = timestamp_to_datetime(self.timestamp) + except AssertionError: + self.when = None if self.when is None: self.when = timezone.now() + # Use the next minute and zero seconds + # The web browser does not select seconds by default + self.when = self.when.replace(second=0) + timezone.timedelta(minutes=1) return super().dispatch(request, *args, **kwargs) def get_initial(self): From 5bf4eeac488cda6881b755780b83daf9b57f9241 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 06:50:58 -0400 Subject: [PATCH 156/174] The split entry doesn't work because you can't strip a list --- tubesync/sync/forms.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 6df609da..2d03e4ea 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -49,10 +49,6 @@ class ScheduleTaskForm(forms.Form): when = forms.DateTimeField( label=_('When the task should run'), required=True, - #widget=forms.SplitDateTimeWidget( - # date_attrs={'type': 'date'}, - # time_attrs={'type': 'time'}, - #), widget=forms.DateTimeInput( attrs={'type': 'datetime-local'}, ), From af14930a077decfa1f76fabd5629a54ba460a16d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 08:16:53 -0400 Subject: [PATCH 157/174] Clean up extra metadata rows --- tubesync/sync/signals.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 657622ac..62f104bb 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -2,6 +2,7 @@ from functools import partial from pathlib import Path from tempfile import TemporaryDirectory from django.conf import settings +from django.db import IntegrityError from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.transaction import on_commit from django.dispatch import receiver @@ -439,9 +440,20 @@ def media_post_delete(sender, instance, **kwargs): skipped_media.skip = True skipped_media.manual_skip = True skipped_media.save() - Metadata.objects.filter( + # Re-use the old metadata if it exists + instance_qs = Metadata.objects.filter( media__isnull=True, site=old_metadata.get(site_field) or 'Youtube', key=skipped_media.key, - ).update(media=skipped_media) + ) + try: + instance_qs.update(media=skipped_media) + except IntegrityError: + # Delete the new metadata + Metadata.objects.filter(media=skipped_media).delete() + try: + instance_qs.update(media=skipped_media) + except IntegrityError: + # Delete the old metadata if it still failed + instance_qs.delete() From eace2bf45b3f3c1317466f893840dc028d6526a7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 13:36:00 -0400 Subject: [PATCH 158/174] Do not allow `run_at` to be in the past --- tubesync/sync/views.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index dcf8e3e7..c89041a3 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1016,15 +1016,18 @@ class TaskScheduleView(FormView, SingleObjectMixin): model = Task errors = dict( invalid_when=_('The type ({}) was incorrect.'), + when_before_now=_('The date and time must be in the future.'), ) def __init__(self, *args, **kwargs): + self.now = timezone.now() self.object = None self.timestamp = None self.when = None super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): + self.now = timezone.now() self.object = self.get_object() self.timestamp = kwargs.get('timestamp') try: @@ -1032,7 +1035,7 @@ class TaskScheduleView(FormView, SingleObjectMixin): except AssertionError: self.when = None if self.when is None: - self.when = timezone.now() + self.when = self.now # Use the next minute and zero seconds # The web browser does not select seconds by default self.when = self.when.replace(second=0) + timezone.timedelta(minutes=1) @@ -1040,11 +1043,13 @@ class TaskScheduleView(FormView, SingleObjectMixin): def get_initial(self): initial = super().get_initial() + initial['now'] = self.now initial['when'] = self.when return initial def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) + data['now'] = self.now data['when'] = self.when return data @@ -1059,24 +1064,28 @@ class TaskScheduleView(FormView, SingleObjectMixin): def form_valid(self, form): max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) - now = timezone.now() when = form.cleaned_data.get('when') - if not isinstance(when, now.__class__): + if not isinstance(when, self.now.__class__): form.add_error( 'when', ValidationError( - errors['invalid_when'].format( + self.errors['invalid_when'].format( type(when), ), ), ) + if when < self.now: + form.add_error( + 'when', + ValidationError(self.errors['when_before_now']), + ) if form.errors: return super().form_invalid(form) self.object.attempts = max_attempts // 2 - self.object.run_at = when + self.object.run_at = max(self.now, when) self.object.save() return super().form_valid(form) From 8be28b25bd5de60a3aec11dbdfe2da0724b408df Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 13:46:34 -0400 Subject: [PATCH 159/174] Add a `now` input --- tubesync/sync/forms.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 2d03e4ea..39b1b606 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -46,6 +46,17 @@ class ResetTasksForm(forms.Form): class ScheduleTaskForm(forms.Form): + now = forms.DateTimeField( + label=_('The current date and time'), + required=False, + widget=forms.DateTimeInput( + attrs={ + 'type': 'datetime-local', + 'readonly': 'true', + }, + ), + ) + when = forms.DateTimeField( label=_('When the task should run'), required=True, From 01a44f74cf2ba585066e521b6319e03f2c16884b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 13 May 2025 04:52:10 -0400 Subject: [PATCH 160/174] Unpin Django --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 2976db2e..bf53b4bf 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,7 @@ verify_ssl = true autopep8 = "*" [packages] -django = "<5.2" +django = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" From 25af3e340732dc234f2f97c319a2c947ce319342 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 13 May 2025 05:24:22 -0400 Subject: [PATCH 161/174] Bump version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 391a209d..68bca98b 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.1' +VERSION = '0.15.2' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From aa27e1c22b6db95ce85bc105b1aec353a3b9f3e5 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:09:09 -0400 Subject: [PATCH 162/174] Adjust Python versions --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9edc7225..e3fee696 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python-version }} @@ -102,7 +102,7 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* - name: Run Django tests - run: cd tubesync && python3 manage.py test --verbosity=2 + run: cd tubesync && python3 -W error manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From 6c10a1da5b6ac7809a897b633c4dd61b908e021c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:38:16 -0400 Subject: [PATCH 163/174] Check the Django version before using a removed setting --- tubesync/tubesync/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 68bca98b..06853cbf 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,3 +1,4 @@ +from django import VERSION as DJANGO_VERSION from pathlib import Path from common.utils import getenv @@ -99,7 +100,9 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' TIME_ZONE = getenv('TZ', 'UTC') USE_I18N = True -USE_L10N = True +# Removed in Django 5.0 +if DJANGO_VERSION[0:3] < (5, 0, 0): + USE_L10N = True USE_TZ = True From 406ba8a8cd0728840860bb67e1a9e85ebb268ddd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:47:32 -0400 Subject: [PATCH 164/174] Set `USE_L10N` to `True` only before Django 4 --- tubesync/tubesync/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 06853cbf..7f5922ae 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -100,8 +100,9 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' TIME_ZONE = getenv('TZ', 'UTC') USE_I18N = True -# Removed in Django 5.0 -if DJANGO_VERSION[0:3] < (5, 0, 0): +# Removed in Django 5.0, set to True by default in Django 4.0 +# https://docs.djangoproject.com/en/4.1/releases/4.0/#localization +if DJANGO_VERSION[0:3] < (4, 0, 0): USE_L10N = True USE_TZ = True From 49340701830bd4fa7b5005f5167c5e4549ae38c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:58:42 -0400 Subject: [PATCH 165/174] Explicitly pass `assume_scheme` --- tubesync/sync/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 39b1b606..655b93be 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -12,7 +12,8 @@ class ValidateSourceForm(forms.Form): ) source_url = forms.URLField( label=_('Source URL'), - required=True + required=True, + assume_scheme='http', # Silence RemovedInDjango60Warning ) From b6466c9b0eb4bf818e04bafad5de92ad2a5c7dc9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:12:58 -0400 Subject: [PATCH 166/174] Always use the UTC time zone --- tubesync/common/timestamp.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index df7b2f13..d8b69178 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -1,8 +1,8 @@ import datetime -posix_epoch = datetime.datetime.utcfromtimestamp(0) utc_tz = datetime.timezone.utc +posix_epoch = datetime.datetime.fromtimestamp(0, utc_tz) def add_epoch(seconds): @@ -13,10 +13,9 @@ def add_epoch(seconds): def subtract_epoch(arg_dt, /): assert isinstance(arg_dt, datetime.datetime) - epoch = posix_epoch.astimezone(utc_tz) utc_dt = arg_dt.astimezone(utc_tz) - return utc_dt - epoch + return utc_dt - posix_epoch def datetime_to_timestamp(arg_dt, /, *, integer=True): timestamp = subtract_epoch(arg_dt).total_seconds() From 0cf25a1bc8491ad58e4a4e9204e2267f025b37fa Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:29:05 -0400 Subject: [PATCH 167/174] `assume_scheme` was added in Django 5 --- tubesync/sync/forms.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 655b93be..e46b740f 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -1,8 +1,14 @@ -from django import forms +from django import forms, VERSION as DJANGO_VERSION from django.utils.translation import gettext_lazy as _ +if DJANGO_VERSION[0:3] < (5, 0, 0): + _assume_scheme = dict() +else: + # Silence RemovedInDjango60Warning + _assume_scheme = dict(assume_scheme='http') + class ValidateSourceForm(forms.Form): source_type = forms.CharField( @@ -13,7 +19,7 @@ class ValidateSourceForm(forms.Form): source_url = forms.URLField( label=_('Source URL'), required=True, - assume_scheme='http', # Silence RemovedInDjango60Warning + **_assume_scheme, ) From b901f6f08c925d1ebb5a09bd68721ce3ba0e8aba Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:39:45 -0400 Subject: [PATCH 168/174] Run `collectstatic` before `test` --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3fee696..27240f3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -101,8 +101,9 @@ jobs: cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* + cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Run Django tests - run: cd tubesync && python3 -W error manage.py test --verbosity=2 + run: cd tubesync && python3 -W default manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From b99cb61cdb47758e9fa9579214092f032f1f651d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:58:29 -0400 Subject: [PATCH 169/174] Don't write `.pyc` files --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 27240f3e..f2a341b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Run Django tests - run: cd tubesync && python3 -W default manage.py test --verbosity=2 + run: cd tubesync && python3 -B -W default manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From 06591a002075dfebe4b8e8a188ca08880a37ba6e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 12:46:57 -0400 Subject: [PATCH 170/174] Remove `Path.is_relative_to` patch for Python `3.8` --- tubesync/sync/signals.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 62f104bb..790ce1c2 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -22,20 +22,6 @@ from .filtering import filter_media from .choices import Val, YouTube_SourceType -def is_relative_to(self, *other): - """Return True if the path is relative to another path or False. - """ - try: - self.relative_to(*other) - return True - except ValueError: - return False - -# patch Path for Python 3.8 -if not hasattr(Path, 'is_relative_to'): - Path.is_relative_to = is_relative_to - - @receiver(pre_save, sender=Source) def source_pre_save(sender, instance, **kwargs): # Triggered before a source is saved, if the schedule has been updated recreate From 1c078322ef44e97a61136e044291870cadad6881 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 12:49:48 -0400 Subject: [PATCH 171/174] Remove Python `3.9` as unsupported --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f2a341b8..b7de46be 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python-version }} From 1142fb6844b55ff84c397a044992564b6ecb42ee Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 13:04:58 -0400 Subject: [PATCH 172/174] Update README.md --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 502abf3a..2ea83c54 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ services: ## Optional authentication -Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and +Available in `v1.0` (or `:latest`) and later. If you want to enable a basic username and password to be required to access the TubeSync dashboard you can set them with the following environment variables: @@ -188,6 +188,14 @@ $ docker pull ghcr.io/meeb/tubesync:v[number] Back-end updates such as database migrations should be automatic. +> [!IMPORTANT] +> `MariaDB` was not automatically upgraded for `UUID` column types. +> To see what changes are needed, you can run: +> ```bash +> docker exec -it tubesync python3 /app/manage.py fix-mariadb --dry-run --uuid-columns +> ``` +> Removing the `--dry-run` will attempt to execute those statements using the configured database connection. + # Moving, backing up, etc. @@ -349,7 +357,7 @@ and you can probably break things by playing in the admin. If you still want to it you can run: ```bash -$ docker exec -ti tubesync python3 /app/manage.py createsuperuser +$ docker exec -it tubesync python3 /app/manage.py createsuperuser ``` And follow the instructions to create an initial Django superuser, once created, you @@ -415,7 +423,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.10) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From be1617979f44d8703176a111dc4c22cec6543ae9 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 15 May 2025 04:09:14 -0400 Subject: [PATCH 173/174] Use `db.connection.ensure_connection()` --- tubesync/sync/management/commands/fix-mariadb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index c3f2287a..58ad2202 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -24,6 +24,7 @@ def SQLTable(arg_table): needle = arg_table if needle.startswith('new__'): needle = arg_table[len('new__'):] + db.connection.ensure_connection() valid_table_name = ( needle in new_tables and arg_table in db_tables(include_views=False) @@ -122,6 +123,7 @@ class Command(BaseCommand): + f': {db.connection.vendor}' ) + db.connection.ensure_connection() db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and From add42bd87c3f90d08750ad3d02ad12e0d261ecce Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 15 May 2025 04:43:59 -0400 Subject: [PATCH 174/174] Use a migration that gives the output we search for --- tubesync/sync/management/commands/fix-mariadb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 58ad2202..9b21d2df 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -222,7 +222,7 @@ class Command(BaseCommand): at_31, err_31, out_31 = check_migration_status( '0031_metadata_metadataformat' ) at_31s, err_31s, out_31s = check_migration_status( '0031_squashed_metadata_metadataformat' ) after_31, err_31a, out_31a = check_migration_status( - '0030_alter_source_source_vcodec', + '0031_metadata_metadataformat', needle='Undo Rename table for metadata to sync_media_metadata', )