From 1e793dbcb94e6f0dc8e98f1b054cdcfbdf876bc9 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 23 Feb 2025 00:21:11 -0500 Subject: [PATCH 001/454] Refactor `make_request` --- tubesync/sync/mediaservers.py | 72 ++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index 3b8e558e..9a793ff2 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -18,14 +18,53 @@ class MediaServer: TIMEOUT = 0 HELP = '' + default_headers = {'User-Agent': 'TubeSync'} def __init__(self, mediaserver_instance): self.object = mediaserver_instance + self.headers = dict(**self.default_headers) + self.token = None + + def make_request_args(self, uri='/', token_header=None, headers={}, token_param=None, params={}): + base_parts = urlsplit(self.object.url) + if self.token is None: + self.token = self.object.loaded_options['token'] or None + if token_header and self.token: + headers.update({token_header: self.token}) + self.headers.update(headers) + if token_param and self.token: + params.update({token_param: self.token}) + qs = urlencode(params) + enable_verify = ( + base_parts.scheme.endswith('s') and + self.object.verify_https + ) + url = urlunsplit((base_parts.scheme, base_parts.netloc, uri, qs, '')) + return (url, + { + headers=self.headers, + verify=enable_verify, + timeout=self.TIMEOUT, + }) + + def make_request(self, uri='/', headers={}, params={}): + ''' + A very simple implementation is: + url, kwargs = self.make_request_args(uri=uri, headers=headers, params=params) + return requests.get(url, **kwargs) + ''' + raise NotImplementedError('MediaServer.make_request() must be implemented') def validate(self): + ''' + Called to check that the configured media server values are correct. + ''' raise NotImplementedError('MediaServer.validate() must be implemented') def update(self): + ''' + Called after the `Media` instance has saved a downloaded file. + ''' raise NotImplementedError('MediaServer.update() must be implemented') @@ -48,30 +87,23 @@ class PlexMediaServer(MediaServer): 'here

.') - def make_request(self, uri='/', params={}): - headers = {'User-Agent': 'TubeSync'} - token = self.object.loaded_options['token'] - params['X-Plex-Token'] = token - base_parts = urlsplit(self.object.url) - qs = urlencode(params) - url = urlunsplit((base_parts.scheme, base_parts.netloc, uri, qs, '')) - if self.object.verify_https: - log.debug(f'[plex media server] Making HTTP GET request to: {url}') - return requests.get(url, headers=headers, verify=True, - timeout=self.TIMEOUT) + def make_request(self, uri='/', headers={}, params={}): + url, kwargs = self.make_request_args(uri=uri, headers=headers, token_param='X-Plex-Token', params=params) + log.debug(f'[plex media server] Making HTTP GET request to: {url}') + if kwargs['verify']: + return requests.get(url, **kwargs) else: # If not validating SSL, given this is likely going to be for an internal # or private network, that Plex issues certs *.hash.plex.direct and that # the warning won't ever been sensibly seen in the HTTPS logs, hide it with warnings.catch_warnings(): warnings.simplefilter("ignore") - return requests.get(url, headers=headers, verify=False, - timeout=self.TIMEOUT) + return requests.get(url, **kwargs) def validate(self): ''' A Plex server requires a host, port, access token and a comma-separated - list if library IDs. + list of library IDs. ''' # Check all the required values are present if not self.object.host: @@ -175,16 +207,10 @@ class JellyfinMediaServer(MediaServer): '

The token is required for API access. You can generate a token in your Jellyfin user profile settings.

' '

The libraries is a comma-separated list of library IDs in Jellyfin.

') - def make_request(self, uri='/', params={}): - headers = { - 'User-Agent': 'TubeSync', - 'X-Emby-Token': self.object.loaded_options['token'] # Jellyfin uses the same `X-Emby-Token` header as Emby - } - - url = f'{self.object.url}{uri}' + def make_request(self, uri='/', headers={}, params={}): + url, kwargs = self.make_request_args(uri=uri, token_header='X-Emby-Token', headers=headers, params=params) log.debug(f'[jellyfin media server] Making HTTP GET request to: {url}') - - return requests.get(url, headers=headers, verify=self.object.verify_https, timeout=self.TIMEOUT) + return requests.get(url, **kwargs) def validate(self): if not self.object.host: From b06a68decdbe0bd57339464c6f17a5aa8aa8275b Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 23 Feb 2025 01:00:08 -0500 Subject: [PATCH 002/454] fixup: use `dict` for this syntax --- tubesync/sync/mediaservers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index 9a793ff2..cfa5a52b 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -40,12 +40,11 @@ class MediaServer: self.object.verify_https ) url = urlunsplit((base_parts.scheme, base_parts.netloc, uri, qs, '')) - return (url, - { + return (url, dict( headers=self.headers, verify=enable_verify, timeout=self.TIMEOUT, - }) + )) def make_request(self, uri='/', headers={}, params={}): ''' From 0cb641d31bcad8c5d8a2055af805b5277bb18689 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 23 Feb 2025 04:00:23 -0500 Subject: [PATCH 003/454] Add more headers I've been digging around the Emby and Jellyfin source code. Some of what I found out is documented in the comments. --- tubesync/sync/mediaservers.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index cfa5a52b..d07f2f21 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -5,6 +5,7 @@ from django.forms import ValidationError from urllib.parse import urlsplit, urlunsplit, urlencode from django.utils.translation import gettext_lazy as _ from common.logger import log +from django.conf import settings class MediaServerError(Exception): @@ -203,11 +204,27 @@ class JellyfinMediaServer(MediaServer): HELP = _('

To connect your TubeSync server to your Jellyfin Media Server, please enter the details below.

' '

The host can be either an IP address or a valid hostname.

' '

The port should be between 1 and 65536.

' - '

The token is required for API access. You can generate a token in your Jellyfin user profile settings.

' - '

The libraries is a comma-separated list of library IDs in Jellyfin.

') + '

The "API Key" token is required for API access. Your Jellyfin administrator can generate an "API Key" token for use with TubeSync for you.

' + '

The libraries is a comma-separated list of library IDs in Jellyfin. Leave this blank to see a list.

') def make_request(self, uri='/', headers={}, params={}): + headers.update({'Content-Type': 'application/json'}) url, kwargs = self.make_request_args(uri=uri, token_header='X-Emby-Token', headers=headers, params=params) + # From the Emby source code; + # this is the order in which the headers are tried: + # X-Emby-Authorization: ('MediaBrowser'|'Emby') 'Token'=, 'Client'=, 'Version'= + # X-Emby-Token: + # X-MediaBrowser-Token: + # Jellyfin uses 'Authorization' first, + # then optionally falls back to the 'X-Emby-Authorization' header. + # Jellyfin uses (") around values, but not keys in that header. + token = kwargs['headers'].get('X-Emby-Token', None) + if token: + kwargs['headers'].update({ + 'X-MediaBrowser-Token': token, + 'X-Emby-Authorization': f'Emby Token={token}, Client=TubeSync, Version={settings.VERSION!s}', + 'Authorization': f'MediaBrowser Token="{token}", Client="TubeSync", Version="{settings.VERSION!s}"', + }) log.debug(f'[jellyfin media server] Making HTTP GET request to: {url}') return requests.get(url, **kwargs) From 381a52eedab63310a56a9c8df8a99ffb14189d8d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Feb 2025 17:13:49 -0500 Subject: [PATCH 004/454] Add the very beginning of new metadata/formats models --- tubesync/sync/models.py | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index de874687..0da1ff93 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1582,6 +1582,53 @@ class Media(models.Model): pass +class Metadata(models.Model): + ''' + CREATE TABLE IF NOT EXISTS "sync_metadata" AS + SELECT + "uuid" AS "media_id", + "metadata" ->> '$.extractor_key' AS "site", + "metadata" ->> '$.id' AS "key", + datetime("metadata" ->> '$.timestamp', 'unixepoch') AS "uploaded", + datetime("metadata" ->> '$.epoch', 'unixepoch') AS "retrieved", + "metadata" AS "value" + FROM "sync_media" ; + ''' + class Meta: + pass + pass + uuid = models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the metadata') + ) + media = models.ForeignKey( + Media, + # on_delete=models.DO_NOTHING, + related_name='metadata_media', + help_text=_('Media the metadata belongs to') + ) + + +class MetadataFormat(models.Model): + ''' + CREATE TABLE IF NOT EXISTS "sync_metadata_formats" ( + "metadata_id" REFERENCES "sync_metadata" ("rowid") ON DELETE CASCADE, + "key" char(12) NOT NULL, + "num" INTEGER NOT NULL, + "format_id" varchar(20) NOT NULL, + "value" json not null, + UNIQUE("key", "num"), + UNIQUE("key", "format_id") + ); + ''' + class Meta: + pass + pass + + class MediaServer(models.Model): ''' A remote media server, such as a Plex server. From 61853f49b5d1ded7469aab3be3e9ef9d00786c92 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 18:48:11 -0400 Subject: [PATCH 005/454] The transaction is never committed if the worker is killed --- tubesync/sync/signals.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index be848a0a..cd2cfbee 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -158,12 +158,6 @@ def source_pre_delete(sender, instance, **kwargs): priority=1, verbose_name=verbose_name.format(instance.name), ) - # Try to do it all immediately - # If this is killed, the scheduled task should do the work instead. - delete_all_media_for_source.now( - str(instance.pk), - str(instance.name), - ) @receiver(post_delete, sender=Source) @@ -176,11 +170,6 @@ def source_post_delete(sender, instance, **kwargs): delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) - # Remove the directory, if the user requested that - directory_path = Path(source.directory_path) - if (directory_path / '.to_be_removed').is_file(): - log.info(f'Deleting directory for: {source.name}: {directory_path}') - rmtree(directory_path, True) @receiver(task_failed, sender=Task) From 5d92176257da66d563471a306249146410efa9ff Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 18:54:36 -0400 Subject: [PATCH 006/454] Remove the directory, if requested, after deleting `Media` --- tubesync/sync/tasks.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index b3850a32..3f5bfb92 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -11,7 +11,7 @@ import uuid from io import BytesIO from hashlib import sha1 from datetime import datetime, timedelta -from shutil import copyfile +from shutil import copyfile, rmtree from PIL import Image from django.conf import settings from django.core.files.base import ContentFile @@ -719,8 +719,8 @@ def wait_for_media_premiere(media_id): media.title = _(f'Premieres in {hours(media.published - now)} hours') media.save() -@background(schedule=300, remove_existing_tasks=False) -def delete_all_media_for_source(source_id, source_name): +@background(schedule=90, remove_existing_tasks=False) +def delete_all_media_for_source(source_id, source_name, source_directory_path): source = None try: source = Source.objects.get(pk=source_id) @@ -734,8 +734,14 @@ def delete_all_media_for_source(source_id, source_name): ).filter( source=source or source_id, ) - for media in mqs: - log.info(f'Deleting media for source: {source_name} item: {media.name}') - with atomic(): - media.delete() + with atomic(durable=True): + for media in mqs: + log.info(f'Deleting media for source: {source_name} item: {media.name}') + with atomic(): + media.delete() + # Remove the directory, if the user requested that + directory_path = Path(source_directory_path) + if (directory_path / '.to_be_removed').is_file(): + log.info(f'Deleting directory for: {source_name}: {directory_path}') + rmtree(directory_path, True) From d3e544def21f06a97471992c28f34e2bbc81116f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 18 Mar 2025 19:18:02 -0400 Subject: [PATCH 007/454] Schedule the media removal after the transaction succeeded --- tubesync/sync/signals.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index cd2cfbee..7aadbcc6 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -1,8 +1,9 @@ +from functools import partial from pathlib import Path -from shutil import rmtree from tempfile import TemporaryDirectory from django.conf import settings from django.db.models.signals import pre_save, post_save, pre_delete, post_delete +from django.db.transaction import on_commit from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed @@ -142,6 +143,7 @@ def source_post_save(sender, instance, created, **kwargs): def source_pre_delete(sender, instance, **kwargs): # Triggered before a source is deleted, delete all media objects to trigger # the Media models post_delete signal + source = instance log.info(f'Deactivating source: {instance.name}') instance.deactivate() log.info(f'Deleting tasks for source: {instance.name}') @@ -152,12 +154,14 @@ def source_pre_delete(sender, instance, **kwargs): # Schedule deletion of media delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk) verbose_name = _('Deleting all media for source "{}"') - delete_all_media_for_source( - str(instance.pk), - str(instance.name), + on_commit(partial( + delete_all_media_for_source, + str(source.pk), + str(source.name), + source.directory_path, priority=1, - verbose_name=verbose_name.format(instance.name), - ) + verbose_name=verbose_name.format(source.name), + )) @receiver(post_delete, sender=Source) @@ -167,7 +171,6 @@ def source_post_delete(sender, instance, **kwargs): log.info(f'Deleting tasks for removed source: {source.name}') delete_task_by_source('sync.tasks.index_source_task', instance.pk) delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk) - delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) From 2484a30900255b0c7784fdf98e1705d54cf13f21 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 19 Mar 2025 08:37:29 -0400 Subject: [PATCH 008/454] Use `tasks.schedule_media_servers_update` --- .../sync/management/commands/delete-source.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 206aee7f..6aef7039 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -2,19 +2,18 @@ import os import uuid from django.utils.translation import gettext_lazy as _ from django.core.management.base import BaseCommand, CommandError -from django.db.models import signals +from django.db.transaction import atomic from common.logger import log from sync.models import Source, Media, MediaServer -from sync.signals import media_post_delete -from sync.tasks import rescan_media_server +from sync.tasks import schedule_media_servers_update class Command(BaseCommand): - help = ('Deletes a source by UUID') + help = _('Deletes a source by UUID') def add_arguments(self, parser): - parser.add_argument('--source', action='store', required=True, help='Source UUID') + parser.add_argument('--source', action='store', required=True, help=_('Source UUID')) def handle(self, *args, **options): source_uuid_str = options.get('source', '') @@ -30,22 +29,14 @@ class Command(BaseCommand): raise CommandError(f'Source does not exist with ' f'UUID: {source_uuid}') # Reconfigure the source to not update the disk or media servers - source.deactivate() + with atomic(durable=True): + source.deactivate() # Delete the source, triggering pre-delete signals for each media item log.info(f'Found source with UUID "{source.uuid}" with name ' f'"{source.name}" and deleting it, this may take some time!') log.info(f'Source directory: {source.directory_path}') - source.delete() - # Update any media servers - for mediaserver in MediaServer.objects.all(): - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - rescan_media_server( - str(mediaserver.pk), - priority=0, - schedule=30, - verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True - ) + with atomic(durable=True): + source.delete() + schedule_media_servers_update() # All done log.info('Done') From 030b7e87a3fe0c7d213158fa403cf3337958ac6d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 12:17:41 -0400 Subject: [PATCH 009/454] Ensure the directory exists for touch --- tubesync/sync/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 3d1896d2..ec9e9c4a 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -26,7 +26,7 @@ from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ConfirmDeleteMediaServerForm) -from .utils import validate_url, delete_file, multi_key_sort +from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task) @@ -415,6 +415,7 @@ class DeleteSourceView(DeleteView, FormMixin): if delete_media: source = self.get_object() directory_path = pathlib.Path(source.directory_path) + mkdir_p(directory_path) (directory_path / '.to_be_removed').touch(exist_ok=True) return super().post(request, *args, **kwargs) From 64cfb3643aee09d277284d474fbca02c0dc5f925 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 12:33:34 -0400 Subject: [PATCH 010/454] Create missing directory --- tubesync/sync/signals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 7aadbcc6..d3224f9b 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -31,6 +31,7 @@ def source_pre_save(sender, instance, **kwargs): log.debug(f'source_pre_save signal: no existing source: {sender} - {instance}') return + mkdir_p(existing_source.directory_path.resolve(strict=False)) existing_dirpath = existing_source.directory_path.resolve(strict=True) new_dirpath = instance.directory_path.resolve(strict=False) if existing_dirpath != new_dirpath: From f4385b8a5fa7bc8ffe79db3666059aa52785022b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 14:12:31 -0400 Subject: [PATCH 011/454] Pre-convert `Path` for JSON --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index d3224f9b..f401f911 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -159,7 +159,7 @@ def source_pre_delete(sender, instance, **kwargs): delete_all_media_for_source, str(source.pk), str(source.name), - source.directory_path, + str(source.directory_path), priority=1, verbose_name=verbose_name.format(source.name), )) From b223e795e0fb70f710fe742abf7350132386f6e2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 14:16:45 -0400 Subject: [PATCH 012/454] Tweak the variable name --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index c1194d52..cd4ec7ae 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -766,7 +766,7 @@ def wait_for_media_premiere(media_id): media.save() @background(schedule=90, remove_existing_tasks=False) -def delete_all_media_for_source(source_id, source_name, source_directory_path): +def delete_all_media_for_source(source_id, source_name, source_directory): source = None try: source = Source.objects.get(pk=source_id) @@ -786,7 +786,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory_path): with atomic(): media.delete() # Remove the directory, if the user requested that - directory_path = Path(source_directory_path) + directory_path = Path(source_directory) if (directory_path / '.to_be_removed').is_file(): log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From 4c101e49c04e3b1fd8f02200bef5d7129f863b92 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 23:04:16 -0400 Subject: [PATCH 013/454] Hackish solution to the slow deletion of media --- tubesync/sync/views.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index ec9e9c4a..5ed2ba78 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -410,11 +410,38 @@ class DeleteSourceView(DeleteView, FormMixin): context_object_name = 'source' def post(self, request, *args, **kwargs): + source = self.get_object() + media_source = dict( + uuid=None, + index_schedule=IndexSchedule.NEVER, + download_media=False, + index_videos=False, + index_streams=False, + filter_text=str(source.pk), + ) + copy_fields = set(map(lambda f: f.name, source._meta.fields)) - set(media_source.keys()) + for k, v in source.__dict__.items(): + if k in copy_fields: + media_source[k] = v + media_source = Source(**media_source) delete_media_val = request.POST.get('delete_media', False) delete_media = True if delete_media_val is not False else False + # overload this boolean for our own use + media_source.delete_removed_media = delete_media + # adjust the directory and key on the source to be deleted + source.directory = source.directory + '/deleted' + source.key = source.key + '/deleted' + source.name = f'[Deleting] {source.name}' + source.save(update_fields={'directory', 'key', 'name'}) + source.refresh_from_db() + # save the new media source now that it is not a duplicate + media_source.uuid = None + media_source.save() + media_source.refresh_from_db() + # switch the media to the new source instance + Media.objects.filter(source=source).update(source=media_source) if delete_media: - source = self.get_object() - directory_path = pathlib.Path(source.directory_path) + directory_path = pathlib.Path(media_source.directory_path) mkdir_p(directory_path) (directory_path / '.to_be_removed').touch(exist_ok=True) return super().post(request, *args, **kwargs) From 2ae35e421e71040c8e9a12c76855df83134f3872 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 23:30:01 -0400 Subject: [PATCH 014/454] Check the overloaded `delete_removed_media` field --- tubesync/sync/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index cd4ec7ae..7fc27c7d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -787,7 +787,11 @@ def delete_all_media_for_source(source_id, source_name, source_directory): media.delete() # Remove the directory, if the user requested that directory_path = Path(source_directory) - if (directory_path / '.to_be_removed').is_file(): + remove = ( + (source and source.delete_removed_media) or + (directory_path / '.to_be_removed').is_file() + ) + if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From c4ac5f606f4f8d30c72c6f156cb4b72be2cd2c13 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 23:40:00 -0400 Subject: [PATCH 015/454] Delete the source with media attached --- tubesync/sync/signals.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index f401f911..4a3d0ab2 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -152,16 +152,19 @@ def source_pre_delete(sender, instance, **kwargs): delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk) delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk) delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) + + # Fetch the media source + media_source = Source.objects.filter(filter_text=str(source.pk))[0] # Schedule deletion of media - delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk) + delete_task_by_source('sync.tasks.delete_all_media_for_source', media_source.pk) verbose_name = _('Deleting all media for source "{}"') on_commit(partial( delete_all_media_for_source, - str(source.pk), - str(source.name), - str(source.directory_path), + str(media_source.pk), + str(media_source.name), + str(media_source.directory_path), priority=1, - verbose_name=verbose_name.format(source.name), + verbose_name=verbose_name.format(media_source.name), )) From 7eb2470fbd5a46cafe8e2637a88b3bd705d0b817 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Mar 2025 23:48:33 -0400 Subject: [PATCH 016/454] fixup: import `Path` --- tubesync/sync/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 7fc27c7d..9dc46fab 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -10,6 +10,7 @@ import math import uuid from io import BytesIO from hashlib import sha1 +from pathlib import Path from datetime import datetime, timedelta from shutil import copyfile, rmtree from PIL import Image From ee70816c4cd850943b4fb371ffa3a62325312696 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:02:04 -0400 Subject: [PATCH 017/454] Delete the source with media attached --- tubesync/sync/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 9dc46fab..59faa7c9 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -792,6 +792,8 @@ def delete_all_media_for_source(source_id, source_name, source_directory): (source and source.delete_removed_media) or (directory_path / '.to_be_removed').is_file() ) + with atomic(durable=True): + source.delete() if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From 4303237b141b46f0cebc8addedacdb2d73892fa3 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:24:49 -0400 Subject: [PATCH 018/454] The source with media attached won't exist after deletion --- tubesync/sync/signals.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 4a3d0ab2..db37444b 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -154,18 +154,20 @@ def source_pre_delete(sender, instance, **kwargs): delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk) # Fetch the media source - media_source = Source.objects.filter(filter_text=str(source.pk))[0] - # Schedule deletion of media - delete_task_by_source('sync.tasks.delete_all_media_for_source', media_source.pk) - verbose_name = _('Deleting all media for source "{}"') - on_commit(partial( - delete_all_media_for_source, - str(media_source.pk), - str(media_source.name), - str(media_source.directory_path), - priority=1, - verbose_name=verbose_name.format(media_source.name), - )) + sqs = Source.objects.filter(filter_text=str(source.pk)) + if sqs.count(): + media_source = sqs[0] + # Schedule deletion of media + delete_task_by_source('sync.tasks.delete_all_media_for_source', media_source.pk) + verbose_name = _('Deleting all media for source "{}"') + on_commit(partial( + delete_all_media_for_source, + str(media_source.pk), + str(media_source.name), + str(media_source.directory_path), + priority=1, + verbose_name=verbose_name.format(media_source.name), + )) @receiver(post_delete, sender=Source) From 78bd153158d69268b190028a3a4bf1a87bc23a25 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:26:33 -0400 Subject: [PATCH 019/454] Delete the source only when it was found --- tubesync/sync/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 59faa7c9..679c8b75 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -792,8 +792,9 @@ def delete_all_media_for_source(source_id, source_name, source_directory): (source and source.delete_removed_media) or (directory_path / '.to_be_removed').is_file() ) - with atomic(durable=True): - source.delete() + if source: + with atomic(durable=True): + source.delete() if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From 2b9c1cc90202c31cc48a606a0f8029870efba071 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:32:55 -0400 Subject: [PATCH 020/454] fixup: import `IndexSchedule` --- tubesync/sync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 5ed2ba78..f4f9896f 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -30,7 +30,7 @@ from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task) -from .choices import (Val, MediaServerType, SourceResolution, +from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule, YouTube_SourceType, youtube_long_source_types, youtube_help, youtube_validation_urls) from . import signals From f6200a6d78cee401ef9a9613152539be9d56c992 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:45:16 -0400 Subject: [PATCH 021/454] fixup: `is_relative_to` was added in Python 3.9 --- tubesync/sync/signals.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index db37444b..cc7b05d5 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -21,6 +21,20 @@ from .filtering import filter_media from .choices import Val, YouTube_SourceType +def is_relative_to(self, *other): + """Return True if the path is relative to another path or False. + """ + try: + self.relative_to(*other) + return True + except ValueError: + return False + +# patch Path for Python 3.8 +if not hasatrr(Path, 'is_relative_to'): + Path.is_relative_to = is_relative_to + + @receiver(pre_save, sender=Source) def source_pre_save(sender, instance, **kwargs): # Triggered before a source is saved, if the schedule has been updated recreate From e59740733d627d6841d54dde4ce40ce50802b1c4 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 28 Mar 2025 00:47:17 -0400 Subject: [PATCH 022/454] fixup: typo --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index cc7b05d5..97649fe6 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -31,7 +31,7 @@ def is_relative_to(self, *other): return False # patch Path for Python 3.8 -if not hasatrr(Path, 'is_relative_to'): +if not hasattr(Path, 'is_relative_to'): Path.is_relative_to = is_relative_to From 738fca120082bc118c83ecd260f94d3af7011d24 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:01:53 -0400 Subject: [PATCH 023/454] Relaunch the network worker after 12 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run index a9c17d49..60dec45a 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue network + --queue network --duration 43200 From 784acae83ade0afc77de568c509b7110d40fdf3d Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:10:03 -0400 Subject: [PATCH 024/454] Relaunch the database worker after 24 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run index 03b75ea8..f0123c11 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue database + --queue database --duration 86400 --sleep 30 From 7b081a7aafc5b307e7d392c5e3a2c8a4b2ccc0d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:14:42 -0400 Subject: [PATCH 025/454] Relaunch the filesystem worker after 12 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run index 0642054d..c5b3cc85 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue filesystem + --queue filesystem --duration 43200 --sleep 20 From d77ca0f7cc2f4c49db42963b3df00dce04256c90 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:19:38 -0400 Subject: [PATCH 026/454] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run index 60dec45a..7f7bcd26 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue network --duration 43200 + --queue network --duration 43200 \ + --sleep "10.${RANDOM}" From 0011a4ef559e00c340995c51842b8f71302f862b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:20:59 -0400 Subject: [PATCH 027/454] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run index c5b3cc85..c0a9fb79 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue filesystem --duration 43200 --sleep 20 + --queue filesystem --duration 43200 \ + --sleep "20.${RANDOM}" From 045bfcbfd629cd4483384f0cf9ff0eb06db0d33e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:22:09 -0400 Subject: [PATCH 028/454] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run index f0123c11..9fbcbc95 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue database --duration 86400 --sleep 30 + --queue database --duration 86400 \ + --sleep "30.${RANDOM}" From e12aa28a2b63268c00552ec1d6fbbffabe518946 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:33:14 -0400 Subject: [PATCH 029/454] Adjusting `BACKGROUND_TASK_ASYNC_THREADS` is no longer needed --- tubesync/tubesync/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 0ac2b462..c73dbd79 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -212,9 +212,6 @@ if MAX_RUN_TIME < 600: DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 50) -if RENAME_SOURCES or RENAME_ALL_SOURCES: - BACKGROUND_TASK_ASYNC_THREADS += 1 - if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS From 48bca2b996f2db239dc1ef3495be7093a3599591 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:45:44 -0400 Subject: [PATCH 030/454] Enable the `ThreadPool` when `TUBESYNC_WORKERS` is used --- tubesync/tubesync/local_settings.py.container | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index cc20f73b..4f386b66 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -62,6 +62,8 @@ else: DEFAULT_THREADS = 1 BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True) +if BACKGROUND_TASK_ASYNC_THREADS > 1: + BACKGROUND_TASK_RUN_ASYNC = True MEDIA_ROOT = CONFIG_BASE_DIR / 'media' From 30633b1aba9c3280242269d6e9b6d1b9002f14be Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 16:10:07 -0400 Subject: [PATCH 031/454] Update README.md --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 17367a4a..71d37863 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has audio stream (such as music) will download to the `audio` directory. All media with a video stream will be downloaded to the `video` directory. All administration of TubeSync is performed via a web interface. You can optionally add a media server, -currently just Plex, to complete the PVR experience. +currently only Jellyfin or Plex, to complete the PVR experience. # Installation @@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab. ### 3. Media Server updating -Currently TubeSync supports Plex as a media server. You can add your local Plex server +Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server under the "media servers" tab. @@ -234,6 +234,13 @@ view these with: $ docker logs --follow tubesync ``` +To include logs with an issue report, please exteact a file and attach it to the issue. +The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container: + +```bash +docker logs -t tubesync > TubeSync.logs.txt 2>&1 +``` + # Advanced usage guides @@ -371,22 +378,26 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_RENAME_SOURCES | Rename media files from selected sources | Source1_directory,Source2_directory | +| TUBESYNC_RENAME_ALL_SOURCES | Rename media files from all sources | True | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| TUBESYNC_SHRINK_NEW | Filter unneeded information from newly retrieved metadata | True | +| TUBESYNC_SHRINK_OLD | Filter unneeded information from metadata loaded from the database | True | +| TUBESYNC_WORKERS | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2 | +| GUNICORN_WORKERS | Number of `gunicorn` (web request) workers to spawn | 3 | +| LISTEN_HOST | IP address for `gunicorn` to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for `gunicorn` to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | postgresql://user:pass@host:port/database | # Manual, non-containerised, installation @@ -396,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.8) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From c7b9e26ab046418866af91ca45b741a0f4e609ce Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 16:52:46 -0400 Subject: [PATCH 032/454] Add file renaming warning to README.md --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17367a4a..024c915d 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,15 @@ and less common features: # Warnings -### 1. Index frequency +### 1. Automated file renaming +> [!IMPORTANT] +> Currently, file renaming is not enabled by default. +> Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. +> +> To prevent your installation from scheduling media file renaming tasks, +> you must set `TUBESYNC_RENAME_ALL_SOURCES=False` in the environment variables. + +### 2. Index frequency It's a good idea to add sources with as long of an index frequency as possible. This is the duration between indexes of the source. An index is when TubeSync checks to see @@ -258,7 +266,7 @@ what videos available on a channel or playlist to find new media. Try and keep t long as possible, up to 24 hours. -### 2. Indexing massive channels +### 3. Indexing massive channels If you add a massive (several thousand videos) channel to TubeSync and choose "index every hour" or similar short interval it's entirely possible your TubeSync install may From 8bcb5cafdbb52173c78cabaf566ef633d0d019c7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:12:41 -0400 Subject: [PATCH 033/454] Remove unnecessary kwargs --- tubesync/sync/tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 79e283c3..f3daf876 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -208,9 +208,7 @@ def schedule_media_servers_update(): for mediaserver in MediaServer.objects.all(): rescan_media_server( str(mediaserver.pk), - priority=10, verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True, ) @@ -320,7 +318,6 @@ def index_source_task(source_id): verbose_name = _('Downloading metadata for "{}"') download_media_metadata( str(media.pk), - priority=20, verbose_name=verbose_name.format(media.pk), ) # Reset task.verbose_name to the saved value From a62b64c3eee61fdf75462cd289f20371d8bd7945 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:42:30 -0400 Subject: [PATCH 034/454] Adjust FS tasks Move thumbnail download from NET to FS. It does use the network, but not with `yt_dlp` and not in a way that YouTube cares about. --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index f3daf876..0358a6ce 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -479,7 +479,7 @@ def download_media_metadata(media_id): f'{source} / {media}: {media_id}') -@background(schedule=dict(priority=15, run_at=10), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=10, run_at=10), queue=Val(TaskQueue.FS), remove_existing_tasks=True) def download_media_thumbnail(media_id, url): ''' Downloads an image from a URL and save it as a local thumbnail attached to a @@ -654,7 +654,7 @@ def rescan_media_server(mediaserver_id): mediaserver.update() -@background(schedule=dict(priority=25, run_at=600), queue=Val(TaskQueue.FS), remove_existing_tasks=True) +@background(schedule=dict(priority=30, run_at=600), queue=Val(TaskQueue.FS), remove_existing_tasks=True) def save_all_media_for_source(source_id): ''' Iterates all media items linked to a source and saves them to From 3b0ecf29be64addbf3f3932bfb3151c2dd6d3610 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:51:57 -0400 Subject: [PATCH 035/454] Move `refesh_formats` to last position in the NET queue --- tubesync/sync/tasks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0358a6ce..95fe5aba 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -240,7 +240,7 @@ def cleanup_removed_media(source, videos): schedule_media_servers_update() -@background(schedule=dict(priority=10, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=20, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -345,7 +345,7 @@ def check_source_directory_exists(source_id): source.make_directory() -@background(schedule=dict(priority=5, run_at=10), queue=Val(TaskQueue.NET)) +@background(schedule=dict(priority=10, run_at=10), queue=Val(TaskQueue.NET)) def download_source_images(source_id): ''' Downloads an image and save it as a local thumbnail attached to a @@ -395,7 +395,7 @@ def download_source_images(source_id): log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}') -@background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=40, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def download_media_metadata(media_id): ''' Downloads the metadata for a media item. @@ -517,7 +517,7 @@ def download_media_thumbnail(media_id, url): return True -@background(schedule=dict(priority=15, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=30, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def download_media(media_id): ''' Downloads the media to disk and attaches it to the Media instance. @@ -707,7 +707,7 @@ def save_all_media_for_source(source_id): update_task_status(task, None) -@background(schedule=dict(priority=10, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=50, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def refesh_formats(media_id): try: media = Media.objects.get(pk=media_id) From 5b53621360dd17a18808d030918695e086226492 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:28:53 -0400 Subject: [PATCH 036/454] Use the `refesh_formats` task instead of the function --- tubesync/sync/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 95fe5aba..292c244f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -634,7 +634,10 @@ def download_media(media_id): log.error(err) # Try refreshing formats if media.has_metadata: - media.refresh_formats + refesh_formats( + str(media.pk), + verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', + ) # Raising an error here triggers the task to be re-attempted (or fail) raise DownloadFailedException(err) From 8816ba3af04eff939464328653b06544c4ab9eb7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:41:13 -0400 Subject: [PATCH 037/454] refesh_formats => refresh_formats --- tubesync/sync/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 292c244f..3fa4f804 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -54,7 +54,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, - 'sync.tasks.refesh_formats': Media, + 'sync.tasks.refresh_formats': Media, 'sync.tasks.rename_media': Media, 'sync.tasks.rename_all_media_for_source': Source, 'sync.tasks.wait_for_media_premiere': Media, @@ -692,7 +692,7 @@ def save_all_media_for_source(source_id): tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' for mn, media in enumerate(refresh_qs, start=1): update_task_status(task, tvn_format.format(mn)) - refesh_formats( + refresh_formats( str(media.pk), verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', ) @@ -711,7 +711,7 @@ def save_all_media_for_source(source_id): @background(schedule=dict(priority=50, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def refesh_formats(media_id): +def refresh_formats(media_id): try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: From d5e588133414e24972610635bd2e100be2efd20a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:45:44 -0400 Subject: [PATCH 038/454] fixup: missed one `refesh_formats` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3fa4f804..6b9d7a6b 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -634,7 +634,7 @@ def download_media(media_id): log.error(err) # Try refreshing formats if media.has_metadata: - refesh_formats( + refresh_formats( str(media.pk), verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', ) From 8a911d0a7cfea51f452caf554a13b8149856e7eb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 08:25:41 -0400 Subject: [PATCH 039/454] Special case `sqlite` to not use transactions --- tubesync/sync/tasks.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3b02e029..73789caa 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -16,7 +16,7 @@ from PIL import Image from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import DatabaseError, IntegrityError +from django.db import connection, DatabaseError, IntegrityError from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -200,6 +200,16 @@ def migrate_queues(): return qs.update(queue=Val(TaskQueue.NET)) +def save_model(instance): + if 'sqlite' == connection.vendor: + # a transaction here causes too many + # database is locked errors + instance.save() + else: + with atomic(): + instance.save() + + def schedule_media_servers_update(): with atomic(): # Schedule a task to update media servers @@ -261,7 +271,7 @@ def index_source_task(source_id): # Reset any errors # TODO: determine if this affects anything source.has_failed = False - source.save() + save_model(source) # Index the source videos = source.index_media() if not videos: @@ -272,7 +282,7 @@ def index_source_task(source_id): f'is reachable') # Got some media, update the last crawl timestamp source.last_crawl = timezone.now() - source.save() + save_model(source) num_videos = len(videos) log.info(f'Found {num_videos} media items for source: {source}') fields = lambda f, m: m.get_metadata_field(f) @@ -303,7 +313,7 @@ def index_source_task(source_id): if published_dt is not None: media.published = published_dt try: - media.save() + save_model(media) except IntegrityError as e: log.error(f'Index media failed: {source} / {media} with "{e}"') else: @@ -477,7 +487,7 @@ def download_media_metadata(media_id): media.duration = media.metadata_duration # Don't filter media here, the post_save signal will handle that - media.save() + save_model(media) log.info(f'Saved {len(media.metadata)} bytes of metadata for: ' f'{source} / {media}: {media_id}') @@ -606,7 +616,7 @@ def download_media(media_id): media.downloaded_hdr = cformat['is_hdr'] else: media.downloaded_format = 'audio' - media.save() + save_model(media) # If selected, copy the thumbnail over as well if media.source.copy_thumbnails: if not media.thumb_file_exists: @@ -704,8 +714,7 @@ def save_all_media_for_source(source_id): for mn, media in enumerate(mqs, start=1): if media.uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) - with atomic(): - media.save() + save_model(media) # Reset task.verbose_name to the saved value update_task_status(task, None) @@ -722,8 +731,7 @@ def refesh_formats(media_id): log.debug(f'Failed to refresh formats for: {media.source} / {media.key}: {e!s}') pass else: - with atomic(): - media.save() + save_model(media) @background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.FS), remove_existing_tasks=True) @@ -780,17 +788,16 @@ def wait_for_media_premiere(media_id): return now = timezone.now() if media.published < now: + # the download tasks start after the media is saved media.manual_skip = False media.skip = False - # start the download tasks - media.save() else: media.manual_skip = True media.title = _(f'Premieres in {hours(media.published - now)} hours') - media.save() task = get_media_premiere_task(media_id) if task: update_task_status(task, f'available in {hours(media.published - now)} hours') + save_model(media) @background(schedule=dict(priority=1, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=False) def delete_all_media_for_source(source_id, source_name): From a1fa823b180d4502824b82e303fc503cf4d3a606 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 17:18:23 -0400 Subject: [PATCH 040/454] POST to `/Items/{ID}/Refresh` --- tubesync/sync/mediaservers.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index 3b8e558e..23c66074 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -175,16 +175,25 @@ class JellyfinMediaServer(MediaServer): '

The token is required for API access. You can generate a token in your Jellyfin user profile settings.

' '

The libraries is a comma-separated list of library IDs in Jellyfin.

') - def make_request(self, uri='/', params={}): + def make_request(self, uri='/', params={}, *, data={}, json=None, method='GET'): headers = { 'User-Agent': 'TubeSync', 'X-Emby-Token': self.object.loaded_options['token'] # Jellyfin uses the same `X-Emby-Token` header as Emby } + assert method in {'GET', 'POST'}, f'Unimplemented method: {method}' url = f'{self.object.url}{uri}' - log.debug(f'[jellyfin media server] Making HTTP GET request to: {url}') + log.debug(f'[jellyfin media server] Making HTTP {method} request to: {url}') - return requests.get(url, headers=headers, verify=self.object.verify_https, timeout=self.TIMEOUT) + return requests.request( + method, url, + headers=headers, + params=params, + data=data, + json=json, + verify=self.object.verify_https, + timeout=self.TIMEOUT, + ) def validate(self): if not self.object.host: @@ -245,8 +254,8 @@ class JellyfinMediaServer(MediaServer): def update(self): libraries = self.object.loaded_options.get('libraries', '').split(',') for library_id in map(str.strip, libraries): - uri = f'/Library/{library_id}/Refresh' - response = self.make_request(uri) + uri = f'/Items/{library_id}/Refresh' + response = self.make_request(uri, method='POST') if response.status_code != 204: # 204 No Content is expected for successful refresh raise MediaServerError(f'Failed to refresh Jellyfin library "{library_id}", status code: {response.status_code}') return True From 40144a8400a14adefc94829fb690093fe282fc2e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 18:41:03 -0400 Subject: [PATCH 041/454] Import the `check_source_directory_exists` task --- tubesync/sync/management/commands/reset-tasks.py | 2 +- tubesync/sync/views.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index 3d6f515d..55436863 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -3,7 +3,7 @@ from django.db.transaction import atomic from django.utils.translation import gettext_lazy as _ from background_task.models import Task from sync.models import Source -from sync.tasks import index_source_task +from sync.tasks import index_source_task, check_source_directory_exists from common.logger import log diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 0e3f8dbb..eeb79ed0 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -29,7 +29,8 @@ from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMedia from .utils import validate_url, delete_file, multi_key_sort from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, - delete_task_by_media, index_source_task, migrate_queues) + delete_task_by_media, index_source_task, + check_source_directory_exists, migrate_queues) from .choices import (Val, MediaServerType, SourceResolution, YouTube_SourceType, youtube_long_source_types, youtube_help, youtube_validation_urls) From f4aef97e1703ad4b79e513d5379c6b0fcc59b838 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 23:05:53 -0400 Subject: [PATCH 042/454] POST to `/Items/{ID}/Refresh` --- tubesync/sync/mediaservers.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index d07f2f21..ce49ff78 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -47,7 +47,7 @@ class MediaServer: timeout=self.TIMEOUT, )) - def make_request(self, uri='/', headers={}, params={}): + def make_request(self, uri='/', /, *, headers={}, params={}): ''' A very simple implementation is: url, kwargs = self.make_request_args(uri=uri, headers=headers, params=params) @@ -87,7 +87,7 @@ class PlexMediaServer(MediaServer): 'here

.') - def make_request(self, uri='/', headers={}, params={}): + def make_request(self, uri='/', /, *, headers={}, params={}): url, kwargs = self.make_request_args(uri=uri, headers=headers, token_param='X-Plex-Token', params=params) log.debug(f'[plex media server] Making HTTP GET request to: {url}') if kwargs['verify']: @@ -207,7 +207,9 @@ class JellyfinMediaServer(MediaServer): '

The "API Key" token is required for API access. Your Jellyfin administrator can generate an "API Key" token for use with TubeSync for you.

' '

The libraries is a comma-separated list of library IDs in Jellyfin. Leave this blank to see a list.

') - def make_request(self, uri='/', headers={}, params={}): + def make_request(self, uri='/', /, *, headers={}, params={}, data={}, json=None, method='GET'): + assert method in {'GET', 'POST'}, f'Unimplemented method: {method}' + headers.update({'Content-Type': 'application/json'}) url, kwargs = self.make_request_args(uri=uri, token_header='X-Emby-Token', headers=headers, params=params) # From the Emby source code; @@ -222,11 +224,17 @@ class JellyfinMediaServer(MediaServer): if token: kwargs['headers'].update({ 'X-MediaBrowser-Token': token, - 'X-Emby-Authorization': f'Emby Token={token}, Client=TubeSync, Version={settings.VERSION!s}', - 'Authorization': f'MediaBrowser Token="{token}", Client="TubeSync", Version="{settings.VERSION!s}"', + 'X-Emby-Authorization': f'Emby Token={token}, Client=TubeSync, Version={settings.VERSION}', + 'Authorization': f'MediaBrowser Token="{token}", Client="TubeSync", Version="{settings.VERSION}"', }) - log.debug(f'[jellyfin media server] Making HTTP GET request to: {url}') - return requests.get(url, **kwargs) + + log.debug(f'[jellyfin media server] Making HTTP {method} request to: {url}') + return requests.request( + method, url, + data=data, + json=json, + **kwargs, + ) def validate(self): if not self.object.host: @@ -287,8 +295,8 @@ class JellyfinMediaServer(MediaServer): def update(self): libraries = self.object.loaded_options.get('libraries', '').split(',') for library_id in map(str.strip, libraries): - uri = f'/Library/{library_id}/Refresh' - response = self.make_request(uri) + uri = f'/Items/{library_id}/Refresh' + response = self.make_request(uri, method='POST') if response.status_code != 204: # 204 No Content is expected for successful refresh raise MediaServerError(f'Failed to refresh Jellyfin library "{library_id}", status code: {response.status_code}') return True From 19f277e6289e1ebacffc3b5aaa1193edac2971b9 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 13:07:13 -0400 Subject: [PATCH 043/454] Create restart_services.sh --- tubesync/restart_services.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tubesync/restart_services.sh diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh new file mode 100644 index 00000000..140bc6d9 --- /dev/null +++ b/tubesync/restart_services.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +if [ 0 -eq $# ] +then + set -- \ + /run/service/tubesync*-worker \ + /run/service/gunicorn \ + /run/service/nginx +fi + +for service in "$@" +do + printf 1>&2 -- 'Restarting %s... ' "${service}" + /command/s6-svc -wr -r "${service}" + printf 1>&2 -- 'completed.\n' +done +unset -v service From 09fea177918cd37713d176b6da73e4078ba9532f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 13:30:17 -0400 Subject: [PATCH 044/454] Resolve paths for me from service names --- tubesync/restart_services.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh index 140bc6d9..e3da89ef 100644 --- a/tubesync/restart_services.sh +++ b/tubesync/restart_services.sh @@ -1,17 +1,23 @@ #!/usr/bin/env sh +dir='/run/service' +svc_path() ( + cd "${dir}" + realpath -e -s "$@" +) + if [ 0 -eq $# ] then set -- \ - /run/service/tubesync*-worker \ - /run/service/gunicorn \ - /run/service/nginx + $( cd "${dir}" && svc_path tubesync*-worker ) \ + "$( svc_path gunicorn )" \ + "$( svc_path nginx )" fi -for service in "$@" +for service in $( svc_path "$@" ) do - printf 1>&2 -- 'Restarting %s... ' "${service}" + printf -- 'Restarting %s...' "${service#${dir}/}" /command/s6-svc -wr -r "${service}" - printf 1>&2 -- 'completed.\n' + printf -- '\tcompleted.\n' done unset -v service From b03a1a3623d1694d8ea6c128ae52fdb03e0ac254 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 13:45:30 -0400 Subject: [PATCH 045/454] Time and report on the `s6-svc` command --- tubesync/restart_services.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh index e3da89ef..b89081f8 100644 --- a/tubesync/restart_services.sh +++ b/tubesync/restart_services.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash dir='/run/service' svc_path() ( @@ -17,7 +17,10 @@ fi for service in $( svc_path "$@" ) do printf -- 'Restarting %s...' "${service#${dir}/}" + _began="$(date '+%s')" /command/s6-svc -wr -r "${service}" - printf -- '\tcompleted.\n' + _ended="$(date '+%s')" + printf -- '\tcompleted (in %d seconds).\n' \ + "$(( "${_ended}" - "${_began}" ))" done -unset -v service +unset -v _began _ended service From 66d8d5eec9b30dced5809efc14a107055f1da3d4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 14:24:23 -0400 Subject: [PATCH 046/454] `dash` is happy when we use `expr` --- tubesync/restart_services.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh index b89081f8..e3d94027 100644 --- a/tubesync/restart_services.sh +++ b/tubesync/restart_services.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh dir='/run/service' svc_path() ( @@ -17,10 +17,10 @@ fi for service in $( svc_path "$@" ) do printf -- 'Restarting %s...' "${service#${dir}/}" - _began="$(date '+%s')" + _began="$( date '+%s' )" /command/s6-svc -wr -r "${service}" - _ended="$(date '+%s')" + _ended="$( date '+%s' )" printf -- '\tcompleted (in %d seconds).\n' \ - "$(( "${_ended}" - "${_began}" ))" + "$( expr "${_ended}" - "${_began}" )" done unset -v _began _ended service From edd3c2767a672a69c19c5977e4427cf8bbc5d41b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 14:44:25 -0400 Subject: [PATCH 047/454] Nicer formatting for the output --- tubesync/restart_services.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh index e3d94027..bb34d809 100644 --- a/tubesync/restart_services.sh +++ b/tubesync/restart_services.sh @@ -16,11 +16,11 @@ fi for service in $( svc_path "$@" ) do - printf -- 'Restarting %s...' "${service#${dir}/}" + printf -- 'Restarting %-28s' "${service#${dir}/}..." _began="$( date '+%s' )" /command/s6-svc -wr -r "${service}" _ended="$( date '+%s' )" - printf -- '\tcompleted (in %d seconds).\n' \ + printf -- '\tcompleted (in %2.1d seconds).\n' \ "$( expr "${_ended}" - "${_began}" )" done unset -v _began _ended service From 91c7608ea2b91508743700b940ed5cfeecde1466 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 14:46:23 -0400 Subject: [PATCH 048/454] Set executable bit --- tubesync/restart_services.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tubesync/restart_services.sh diff --git a/tubesync/restart_services.sh b/tubesync/restart_services.sh old mode 100644 new mode 100755 From 580a77a5c8986326355870d669c18e8d387ddfb8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 10 Apr 2025 19:01:03 -0400 Subject: [PATCH 049/454] Only hide warnings when using `https` When not verifying certificates and using `https` we want to hide warnings about the certificate. Otherwise, we want to not hide anything. --- tubesync/sync/mediaservers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index ce49ff78..e0f9e7e7 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -90,15 +90,14 @@ class PlexMediaServer(MediaServer): def make_request(self, uri='/', /, *, headers={}, params={}): url, kwargs = self.make_request_args(uri=uri, headers=headers, token_param='X-Plex-Token', params=params) log.debug(f'[plex media server] Making HTTP GET request to: {url}') - if kwargs['verify']: - return requests.get(url, **kwargs) - else: + if self.object.use_https and not kwargs['verify']: # If not validating SSL, given this is likely going to be for an internal # or private network, that Plex issues certs *.hash.plex.direct and that # the warning won't ever been sensibly seen in the HTTPS logs, hide it with warnings.catch_warnings(): warnings.simplefilter("ignore") return requests.get(url, **kwargs) + return requests.get(url, **kwargs) def validate(self): ''' @@ -229,6 +228,16 @@ class JellyfinMediaServer(MediaServer): }) log.debug(f'[jellyfin media server] Making HTTP {method} request to: {url}') + if self.object.use_https and not kwargs['verify']: + # not verifying certificates + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return requests.request( + method, url, + data=data, + json=json, + **kwargs, + ) return requests.request( method, url, data=data, From 117e7c2eef7f83da3e0b2a90c73b1f0de9d37a78 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 00:58:45 -0400 Subject: [PATCH 050/454] `3.9` is a better minimum Python version The parser change and a few other things mean that compatibility with `3.8` is unlikely to last much longer. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71d37863..5fc96c4b 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.8) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From 4c6b4e3c73e9b2aa46eda41345b6300c2812044e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 01:02:49 -0400 Subject: [PATCH 051/454] Update README.md --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 17367a4a..5fc96c4b 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has audio stream (such as music) will download to the `audio` directory. All media with a video stream will be downloaded to the `video` directory. All administration of TubeSync is performed via a web interface. You can optionally add a media server, -currently just Plex, to complete the PVR experience. +currently only Jellyfin or Plex, to complete the PVR experience. # Installation @@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab. ### 3. Media Server updating -Currently TubeSync supports Plex as a media server. You can add your local Plex server +Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server under the "media servers" tab. @@ -234,6 +234,13 @@ view these with: $ docker logs --follow tubesync ``` +To include logs with an issue report, please exteact a file and attach it to the issue. +The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container: + +```bash +docker logs -t tubesync > TubeSync.logs.txt 2>&1 +``` + # Advanced usage guides @@ -371,22 +378,26 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_RENAME_SOURCES | Rename media files from selected sources | Source1_directory,Source2_directory | +| TUBESYNC_RENAME_ALL_SOURCES | Rename media files from all sources | True | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| TUBESYNC_SHRINK_NEW | Filter unneeded information from newly retrieved metadata | True | +| TUBESYNC_SHRINK_OLD | Filter unneeded information from metadata loaded from the database | True | +| TUBESYNC_WORKERS | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2 | +| GUNICORN_WORKERS | Number of `gunicorn` (web request) workers to spawn | 3 | +| LISTEN_HOST | IP address for `gunicorn` to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for `gunicorn` to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | postgresql://user:pass@host:port/database | # Manual, non-containerised, installation @@ -396,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From ba63cece20bbbdd2582c529540ec0ed269b9941b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 01:08:00 -0400 Subject: [PATCH 052/454] Revert changes to `README.md` --- README.md | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 5fc96c4b..17367a4a 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has audio stream (such as music) will download to the `audio` directory. All media with a video stream will be downloaded to the `video` directory. All administration of TubeSync is performed via a web interface. You can optionally add a media server, -currently only Jellyfin or Plex, to complete the PVR experience. +currently just Plex, to complete the PVR experience. # Installation @@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab. ### 3. Media Server updating -Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server +Currently TubeSync supports Plex as a media server. You can add your local Plex server under the "media servers" tab. @@ -234,13 +234,6 @@ view these with: $ docker logs --follow tubesync ``` -To include logs with an issue report, please exteact a file and attach it to the issue. -The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container: - -```bash -docker logs -t tubesync > TubeSync.logs.txt 2>&1 -``` - # Advanced usage guides @@ -378,26 +371,22 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_RENAME_SOURCES | Rename media files from selected sources | Source1_directory,Source2_directory | -| TUBESYNC_RENAME_ALL_SOURCES | Rename media files from all sources | True | -| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | -| TUBESYNC_SHRINK_NEW | Filter unneeded information from newly retrieved metadata | True | -| TUBESYNC_SHRINK_OLD | Filter unneeded information from metadata loaded from the database | True | -| TUBESYNC_WORKERS | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2 | -| GUNICORN_WORKERS | Number of `gunicorn` (web request) workers to spawn | 3 | -| LISTEN_HOST | IP address for `gunicorn` to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for `gunicorn` to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | postgresql://user:pass@host:port/database | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | # Manual, non-containerised, installation @@ -407,7 +396,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From 81dbb8f8349eb62b890fdd1de240ec4456d482c5 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 01:13:48 -0400 Subject: [PATCH 053/454] Remove `BACKGROUND_TASK_ASYNC_THREADS` adjustment --- tubesync/tubesync/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index fdf42c3a..37ca2699 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -212,9 +212,6 @@ if MAX_RUN_TIME < 600: DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 50) -if RENAME_SOURCES or RENAME_ALL_SOURCES: - BACKGROUND_TASK_ASYNC_THREADS += 1 - if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS From 50813482258e074dce80f6bfd4ed6c2f370a88e1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 03:18:43 -0400 Subject: [PATCH 054/454] Added the basic columns for metadata and formats --- tubesync/sync/models.py | 138 +++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 24 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 0da1ff93..bca6dfb9 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1584,49 +1584,139 @@ class Media(models.Model): class Metadata(models.Model): ''' - CREATE TABLE IF NOT EXISTS "sync_metadata" AS - SELECT - "uuid" AS "media_id", - "metadata" ->> '$.extractor_key' AS "site", - "metadata" ->> '$.id' AS "key", - datetime("metadata" ->> '$.timestamp', 'unixepoch') AS "uploaded", - datetime("metadata" ->> '$.epoch', 'unixepoch') AS "retrieved", - "metadata" AS "value" - FROM "sync_media" ; + Metadata for an indexed `Media` item. ''' class Meta: - pass - pass + verbose_name = _('Metadata about a Media item') + verbose_name_plural = _('Metadata about a Media item') + unique_together = ( + ('media', 'site', 'key'), + ) + uuid = models.UUIDField( _('uuid'), primary_key=True, editable=False, default=uuid.uuid4, - help_text=_('UUID of the metadata') + help_text=_('UUID of the metadata'), ) media = models.ForeignKey( Media, # on_delete=models.DO_NOTHING, + on_delete=models.CASCADE, related_name='metadata_media', - help_text=_('Media the metadata belongs to') + help_text=_('Media the metadata belongs to'), + null=False, + ) + site = models.CharField( + _('site'), + max_length=256, + blank=True, + null=False, + default='Youtube', + help_text=_('Site from which the metadata was retrieved'), + ) + key = models.CharField( + _('key'), + max_length=256, + blank=True, + null=False, + default='', + help_text=_('Media identifier at the site from which the metadata was retrieved'), + ) + created = models.DateTimeField( + _('created'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was created'), + ) + retrieved = models.DateTimeField( + _('retrieved'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was retrieved'), + ) + uploaded = models.DateTimeField( + _('uploaded'), + null=True, + help_text=_('Date and time the media was uploaded'), + ) + published = models.DateTimeField( + _('published'), + null=True, + help_text=_('Date and time the media was published'), + ) + value = models.JSONField( + _('value'), + null=False, + default=dict, + help_text=_('JSON metadata object'), ) class MetadataFormat(models.Model): ''' - CREATE TABLE IF NOT EXISTS "sync_metadata_formats" ( - "metadata_id" REFERENCES "sync_metadata" ("rowid") ON DELETE CASCADE, - "key" char(12) NOT NULL, - "num" INTEGER NOT NULL, - "format_id" varchar(20) NOT NULL, - "value" json not null, - UNIQUE("key", "num"), - UNIQUE("key", "format_id") - ); + A format from the Metadata for an indexed `Media` item. ''' class Meta: - pass - pass + verbose_name = _('Format from the Metadata about a Media item') + verbose_name_plural = _('Formats from the Metadata about a Media item') + unique_together = ( + ('metadata', 'site', 'key', 'number'), + ('metadata', 'site', 'key', 'code'), + ) + + uuid = models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the format'), + ) + metadata = models.ForeignKey( + Metadata, + # on_delete=models.DO_NOTHING, + on_delete=models.CASCADE, + related_name='metadataformat_metadata', + help_text=_('Metadata the format belongs to'), + null=False, + ) + site = models.CharField( + _('site'), + max_length=256, + blank=True, + null=False, + default='Youtube', + help_text=_('Site from which the format is available'), + ) + key = models.CharField( + _('key'), + max_length=256, + blank=True, + null=False, + default='', + help_text=_('Media identifier at the site for which this format is available'), + ) + number = models.PositiveIntegerField( + _('number'), + blank=False, + null=False, + help_text=_('Ordering number for this format') + ) + code = models.CharField( + _('code'), + max_length=64, + blank=True, + null=False, + default='', + help_text=_('Format identification code'), + ) + value = models.JSONField( + _('value'), + null=False, + default=dict, + help_text=_('JSON metadata format object'), + ) class MediaServer(models.Model): From 26dc67503defbaa6f55932b542c1b27934eaa515 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 03:44:58 -0400 Subject: [PATCH 055/454] Create 0031_metadata_metadataformat.py --- .../0031_metadata_metadataformat.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tubesync/sync/migrations/0031_metadata_metadataformat.py diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py new file mode 100644 index 00000000..b33c1fcc --- /dev/null +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -0,0 +1,51 @@ +# Generated by Django 5.1.8 on 2025-04-11 07:36 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0030_alter_source_source_vcodec'), + ] + + operations = [ + migrations.CreateModel( + name='Metadata', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the metadata', primary_key=True, serialize=False, verbose_name='uuid')), + ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was created', verbose_name='created')), + ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), + ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), + ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), + ('value', models.JSONField(default=dict, help_text='JSON metadata object', verbose_name='value')), + ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), + ], + options={ + 'verbose_name': 'Metadata about a Media item', + 'verbose_name_plural': 'Metadata about a Media item', + 'unique_together': {('media', 'site', 'key')}, + }, + ), + migrations.CreateModel( + name='MetadataFormat', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the format', primary_key=True, serialize=False, verbose_name='uuid')), + ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), + ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), + ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), + ('value', models.JSONField(default=dict, help_text='JSON metadata format object', verbose_name='value')), + ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), + ], + options={ + 'verbose_name': 'Format from the Metadata about a Media item', + 'verbose_name_plural': 'Formats from the Metadata about a Media item', + 'unique_together': {('metadata', 'site', 'key', 'code'), ('metadata', 'site', 'key', 'number')}, + }, + ), + ] From 9bf84efb16e83cd63749d63a61d2e0263fbafa25 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 05:50:04 -0400 Subject: [PATCH 056/454] Add JSONEncoder for all iterable objects --- tubesync/sync/models.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index aef64130..7991c77d 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -11,6 +11,7 @@ from django.conf import settings from django.db import models from django.core.exceptions import SuspiciousOperation from django.core.files.storage import FileSystemStorage +from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import RegexValidator from django.utils.text import slugify from django.utils import timezone @@ -35,6 +36,20 @@ from .choices import (Val, CapChoices, Fallback, FileExtension, media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) +class JSONEncoder(DjangoJSONEncoder): + item_separator = ',' + key_separator = ':' + + def default(self, obj): + try: + iterable = iter(obj) + except TypeError: + pass + else: + return list(iterable) + return super().default(obj) + + class Source(models.Model): ''' A Source is a source of media. Currently, this is either a YouTube channel @@ -1747,6 +1762,7 @@ class Metadata(models.Model): ) value = models.JSONField( _('value'), + encoder=JSONEncoder, null=False, default=dict, help_text=_('JSON metadata object'), @@ -1812,6 +1828,7 @@ class MetadataFormat(models.Model): ) value = models.JSONField( _('value'), + encoder=JSONEncoder, null=False, default=dict, help_text=_('JSON metadata format object'), From 74107e15e97ba0bd9426ebe40aa4ab8e3117ea74 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 05:56:37 -0400 Subject: [PATCH 057/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index b33c1fcc..d64596e7 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -22,7 +22,7 @@ class Migration(migrations.Migration): ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), - ('value', models.JSONField(default=dict, help_text='JSON metadata object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), ], options={ @@ -39,7 +39,7 @@ class Migration(migrations.Migration): ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), - ('value', models.JSONField(default=dict, help_text='JSON metadata format object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), ], options={ From 64e4fca7079779a9e74bfa01aa3e4b675e151cff Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 05:59:41 -0400 Subject: [PATCH 058/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index d64596e7..381dace4 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -1,6 +1,7 @@ # Generated by Django 5.1.8 on 2025-04-11 07:36 import django.db.models.deletion +import sync.models import uuid from django.db import migrations, models From ee00cffd76e441bbae8ae4b71e71672758ddfea7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 03:27:17 -0400 Subject: [PATCH 059/454] Try `refresh_formats` for `NoFormatException` --- tubesync/sync/tasks.py | 54 ++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3b02e029..af6e2266 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -24,12 +24,13 @@ from background_task import background from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask from common.logger import log -from common.errors import NoMediaException, NoMetadataException, DownloadFailedException +from common.errors import ( NoFormatException, NoMediaException, + NoMetadataException, DownloadFailedException, ) from common.utils import json_serial, remove_enclosed from .choices import Val, TaskQueue from .models import Source, Media, MediaServer -from .utils import (get_remote_image, resize_image_to_height, delete_file, - write_text_file, filter_response) +from .utils import ( get_remote_image, resize_image_to_height, delete_file, + write_text_file, filter_response, ) from .youtube import YouTubeError @@ -54,7 +55,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, - 'sync.tasks.refesh_formats': Media, + 'sync.tasks.refresh_formats': Media, 'sync.tasks.rename_media': Media, 'sync.tasks.rename_all_media_for_source': Source, 'sync.tasks.wait_for_media_premiere': Media, @@ -566,9 +567,36 @@ def download_media(media_id): f'not downloading') return filepath = media.filepath + container = format_str = None log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"') - format_str, container = media.download_media() - if os.path.exists(filepath): + try: + format_str, container = media.download_media() + except NoFormatException as e: + # Try refreshing formats + if media.has_metadata: + log.debug(f'Scheduling a task to refresh metadata for: {media.key}: "{media.name}"') + refresh_formats( + str(media.pk), + verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', + ) + log.exception(str(e)) + raise + else: + if not os.path.exists(filepath): + # Try refreshing formats + if media.has_metadata: + log.debug(f'Scheduling a task to refresh metadata for: {media.key}: "{media.name}"') + refresh_formats( + str(media.pk), + verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', + ) + # Expected file doesn't exist on disk + err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, ' + f'expected outfile does not exist: {filepath}') + log.error(err) + # Raising an error here triggers the task to be re-attempted (or fail) + raise DownloadFailedException(err) + # Media has been downloaded successfully log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' f'"{filepath}"') @@ -630,16 +658,6 @@ def download_media(media_id): pass # Schedule a task to update media servers schedule_media_servers_update() - else: - # Expected file doesn't exist on disk - err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, ' - f'expected outfile does not exist: {filepath}') - log.error(err) - # Try refreshing formats - if media.has_metadata: - media.refresh_formats - # Raising an error here triggers the task to be re-attempted (or fail) - raise DownloadFailedException(err) @background(schedule=dict(priority=0, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True) @@ -692,7 +710,7 @@ def save_all_media_for_source(source_id): tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' for mn, media in enumerate(refresh_qs, start=1): update_task_status(task, tvn_format.format(mn)) - refesh_formats( + refresh_formats( str(media.pk), verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', ) @@ -711,7 +729,7 @@ def save_all_media_for_source(source_id): @background(schedule=dict(priority=10, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def refesh_formats(media_id): +def refresh_formats(media_id): try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: From c20990fca35fe82eec5fbe6fab998f47a08562ff Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 04:41:38 -0400 Subject: [PATCH 060/454] Do not call `resize_image_to_height` on smaller images --- tubesync/sync/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3b02e029..e7f31581 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -501,9 +501,10 @@ def download_media_thumbnail(media_id, url): width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) i = get_remote_image(url) - log.info(f'Resizing {i.width}x{i.height} thumbnail to ' - f'{width}x{height}: {url}') - i = resize_image_to_height(i, width, height) + if (i.width > width) and (i.height > height): + log.info(f'Resizing {i.width}x{i.height} thumbnail to ' + f'{width}x{height}: {url}') + i = resize_image_to_height(i, width, height) image_file = BytesIO() i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True) image_file.seek(0) From 9421a3e983d38063354e6c48f7e61b70d7e5f881 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 06:20:33 -0400 Subject: [PATCH 061/454] Fix the `can_download` calculation code --- tubesync/sync/signals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 4c332eca..31486ad3 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -212,10 +212,10 @@ def media_post_save(sender, instance, created, **kwargs): if not instance.can_download: instance.can_download = True can_download_changed = True - else: - if instance.can_download: - instance.can_download = False - can_download_changed = True + else: + if instance.can_download: + instance.can_download = False + can_download_changed = True # Recalculate the "skip_changed" flag skip_changed = filter_media(instance) else: From 4441f92193e7cbc595ccde79680ee0d5e01b8a66 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 06:23:56 -0400 Subject: [PATCH 062/454] Fix indentation for signals.py --- tubesync/sync/signals.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 31486ad3..c8e48ca4 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -9,12 +9,12 @@ from background_task.signals import task_failed from background_task.models import Task from common.logger import log from .models import Source, Media, MediaServer -from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, - download_media_thumbnail, download_media_metadata, - map_task_to_instance, check_source_directory_exists, - download_media, rescan_media_server, download_source_images, - delete_all_media_for_source, save_all_media_for_source, - rename_media, get_media_metadata_task, get_media_download_task) +from .tasks import ( delete_task_by_source, delete_task_by_media, index_source_task, + download_media_thumbnail, download_media_metadata, + map_task_to_instance, check_source_directory_exists, + download_media, rescan_media_server, download_source_images, + delete_all_media_for_source, save_all_media_for_source, + rename_media, get_media_metadata_task, get_media_download_task) from .utils import delete_file, glob_quote, mkdir_p from .filtering import filter_media from .choices import Val, YouTube_SourceType From 689ad677c83390bbd6846b85552787b4d7f531b0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 06:36:05 -0400 Subject: [PATCH 063/454] Fix indentation for signals.py --- tubesync/sync/signals.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index c8e48ca4..0310da37 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -9,12 +9,12 @@ from background_task.signals import task_failed from background_task.models import Task from common.logger import log from .models import Source, Media, MediaServer -from .tasks import ( delete_task_by_source, delete_task_by_media, index_source_task, - download_media_thumbnail, download_media_metadata, - map_task_to_instance, check_source_directory_exists, - download_media, rescan_media_server, download_source_images, - delete_all_media_for_source, save_all_media_for_source, - rename_media, get_media_metadata_task, get_media_download_task) +from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, + download_media_thumbnail, download_media_metadata, + map_task_to_instance, check_source_directory_exists, + download_media, rescan_media_server, download_source_images, + delete_all_media_for_source, save_all_media_for_source, + rename_media, get_media_metadata_task, get_media_download_task) from .utils import delete_file, glob_quote, mkdir_p from .filtering import filter_media from .choices import Val, YouTube_SourceType @@ -250,8 +250,10 @@ def media_post_save(sender, instance, created, **kwargs): if not instance.thumb and not instance.skip: thumbnail_url = instance.thumbnail if thumbnail_url: - log.info(f'Scheduling task to download thumbnail for: {instance.name} ' - f'from: {thumbnail_url}') + log.info( + 'Scheduling task to download thumbnail' + f' for: {instance.name} from: {thumbnail_url}' + ) verbose_name = _('Downloading thumbnail for "{}"') download_media_thumbnail( str(instance.pk), @@ -289,8 +291,10 @@ def media_pre_delete(sender, instance, **kwargs): delete_task_by_media('sync.tasks.wait_for_media_premiere', (str(instance.pk),)) thumbnail_url = instance.thumbnail if thumbnail_url: - delete_task_by_media('sync.tasks.download_media_thumbnail', - (str(instance.pk), thumbnail_url)) + delete_task_by_media( + 'sync.tasks.download_media_thumbnail', + (str(instance.pk), thumbnail_url,), + ) # Remove thumbnail file for deleted media if instance.thumb: instance.thumb.delete(save=False) From ef1dd0eba8ab61355273fe5ed4ab85d3a24135db Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 09:22:25 -0400 Subject: [PATCH 064/454] Reduce the memory usage --- tubesync/sync/tasks.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 49a5a36f..0d9705bb 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -674,7 +674,10 @@ def save_all_media_for_source(source_id): raise InvalidTaskError(_('no such source')) from e saved_later = set() - mqs = Media.objects.filter(source=source) + mqs = Media.objects.all().defer( + 'metadata', + 'thumb', + ).filter(source=source) task = get_source_check_task(source_id) refresh_qs = mqs.filter( can_download=False, @@ -701,11 +704,17 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media in enumerate(mqs, start=1): - if media.uuid not in saved_later: + for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True), start=1): + if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) - with atomic(): - media.save() + try: + media = Media.objects.get(pk=str(media_uuid)) + except Media.DoesNotExist as e: + log.exception(str(e)) + pass + else: + with atomic(): + media.save() # Reset task.verbose_name to the saved value update_task_status(task, None) From d4c360eaeeeb91fc81aa708fac668b1a15009297 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 10:35:50 -0400 Subject: [PATCH 065/454] Use Django query set iterator function --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0d9705bb..76461daa 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -693,7 +693,7 @@ def save_all_media_for_source(source_id): end=task.verbose_name.find('Check'), ) tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' - for mn, media in enumerate(refresh_qs, start=1): + for mn, media in enumerate(refresh_qs.iterator(chunk_size=1000), start=1): update_task_status(task, tvn_format.format(mn)) refresh_formats( str(media.pk), @@ -704,7 +704,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True), start=1): + for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True).iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: From 7c8c5e2b41f40ec5cba0f8c171af820c732ab10c Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 12:16:54 -0400 Subject: [PATCH 066/454] Use Django query set iterator function some more --- tubesync/sync/tasks.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 76461daa..2252679d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -205,7 +205,7 @@ def schedule_media_servers_update(): # Schedule a task to update media servers log.info(f'Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all(): + for mediaserver in MediaServer.objects.all().iterator(): rescan_media_server( str(mediaserver.pk), verbose_name=verbose_name.format(mediaserver), @@ -214,9 +214,15 @@ def schedule_media_servers_update(): def cleanup_old_media(): with atomic(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): + for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0).iterator(chunk_size=1000): delta = timezone.now() - timedelta(days=source.days_to_keep) - for media in source.media_source.filter(downloaded=True, download_date__lt=delta): + mqs = source.media_source.defer( + 'metadata', + ).filter( + downloaded=True, + download_date__lt=delta, + ) + for media in mqs.iterator(chunk_size=1000): log.info(f'Deleting expired media: {source} / {media} ' f'(now older than {source.days_to_keep} days / ' f'download_date before {delta})') @@ -230,8 +236,12 @@ def cleanup_removed_media(source, videos): if not source.delete_removed_media: return log.info(f'Cleaning up media no longer in source: {source}') - media_objects = Media.objects.filter(source=source) - for media in media_objects: + mqs = Media.objects.defer( + 'metadata', + ).filter( + source=source, + ) + for media in mqs.iterator(chunk_size=1000): matching_source_item = [video['id'] for video in videos if video['id'] == media.key] if not matching_source_item: log.info(f'{media.name} is no longer in source, removing') @@ -766,13 +776,12 @@ def rename_all_media_for_source(source_id): if not create_rename_tasks: return mqs = Media.objects.all().defer( - 'metadata', 'thumb', ).filter( source=source, downloaded=True, ) - for media in mqs: + for media in mqs.iterator(chunk_size=1000): with atomic(): media.rename_files() @@ -816,7 +825,7 @@ def delete_all_media_for_source(source_id, source_name): ).filter( source=source or source_id, ) - for media in mqs: + for media in mqs.iterator(chunk_size=1000): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): media.delete() From 533dc453f0af90fd5174172074ce43815b1d8fdd Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 15:09:11 -0400 Subject: [PATCH 067/454] Add a `metadata_load` function to ingest JSON strings --- tubesync/sync/models.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 7991c77d..7cd18b38 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -848,11 +848,14 @@ class Media(models.Model): fields = self.METADATA_FIELDS.get(field, {}) return fields.get(self.source.source_type, field) - def get_metadata_first_value(self, iterable, default=None, /): + def get_metadata_first_value(self, iterable, default=None, /, *, arg_dict=None): ''' fetch the first key with a value from metadata ''' + if arg_dict is None: + arg_dict = self.loaded_metadata + assert isinstance(arg_dict, dict), type(arg_dict) # str is an iterable of characters # we do not want to look for each character! if isinstance(iterable, str): @@ -860,7 +863,7 @@ class Media(models.Model): for key in tuple(iterable): # reminder: unmapped fields return the key itself field = self.get_metadata_field(key) - value = self.loaded_metadata.get(field) + value = arg_dict.get(field) # value can be None because: # - None was stored at the key # - the key was not in the dictionary @@ -1094,6 +1097,26 @@ class Media(models.Model): return self.metadata is not None + def metadata_load(self, arg_str='{}'): + data = json.loads(arg_str) or self.loaded_metadata + site = self.get_metadata_first_value('extractor_key', arg_dict=data) + epoch = self.get_metadata_first_value('epoch', arg_dict=data) + epoch_dt = self.metadata_published( epoch ) + release = self.get_metadata_first_value('release_timestamp', arg_dict=data) + release_dt = self.metadata_published( release ) + md = self.metadata_media.get_or_create(site=site, key=self.key)[0] + md.value = data + formats = md.value.pop('formats', list()) + md.retrieved = epoch_dt + md.uploaded = self.published + md.published = release_dt or self.published + md.save() + for number, format in enumerate(formats, start=1): + mdf = md.metadataformat_metadata.get_or_create(site=md.site, key=md.key, code=format.get('format_id'), number=number)[0] + mdf.value = format + mdf.save() + + def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value From 6e27695a62971cefceb882a2213a8b599061975e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 16:43:31 -0400 Subject: [PATCH 068/454] Add `Metadata.ingest_formats` function --- tubesync/sync/models.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 7cd18b38..e931e6ea 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -13,6 +13,7 @@ from django.core.exceptions import SuspiciousOperation from django.core.files.storage import FileSystemStorage from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import RegexValidator +from django.db.transaction import atomic from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -1097,24 +1098,22 @@ class Media(models.Model): return self.metadata is not None + @atomic(durable=False) def metadata_load(self, arg_str='{}'): data = json.loads(arg_str) or self.loaded_metadata site = self.get_metadata_first_value('extractor_key', arg_dict=data) epoch = self.get_metadata_first_value('epoch', arg_dict=data) epoch_dt = self.metadata_published( epoch ) - release = self.get_metadata_first_value('release_timestamp', arg_dict=data) + release = self.get_metadata_first_value(('release_timestamp', 'timestamp',), arg_dict=data) release_dt = self.metadata_published( release ) md = self.metadata_media.get_or_create(site=site, key=self.key)[0] md.value = data - formats = md.value.pop('formats', list()) + formats = md.value.pop(self.get_metadata_field('formats'), list()) md.retrieved = epoch_dt md.uploaded = self.published md.published = release_dt or self.published md.save() - for number, format in enumerate(formats, start=1): - mdf = md.metadataformat_metadata.get_or_create(site=md.site, key=md.key, code=format.get('format_id'), number=number)[0] - mdf.value = format - mdf.save() + md.ingest_formats(formats) def save_to_metadata(self, key, value, /): @@ -1791,6 +1790,13 @@ class Metadata(models.Model): help_text=_('JSON metadata object'), ) + @atomic(durable=False) + def ingest_formats(self, formats=list(), /, *): + for number, format in enumerate(formats, start=1): + mdf = self.metadataformat_metadata.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0] + mdf.value = format + mdf.save() + class MetadataFormat(models.Model): ''' From 2011918b0046c5c232a064ab15b6c561c33ae694 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 16:46:51 -0400 Subject: [PATCH 069/454] fixup: function definition --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e931e6ea..075db40a 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1791,7 +1791,7 @@ class Metadata(models.Model): ) @atomic(durable=False) - def ingest_formats(self, formats=list(), /, *): + def ingest_formats(self, formats=list(), /): for number, format in enumerate(formats, start=1): mdf = self.metadataformat_metadata.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0] mdf.value = format From 71c4d63043f749074d864b494d32d1fbf7430da3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 03:53:07 -0400 Subject: [PATCH 070/454] Iterate over independent query sets --- tubesync/sync/tasks.py | 43 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2252679d..92a95e79 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -121,8 +121,7 @@ def update_task_status(task, status): else: task.verbose_name = f'[{status}] {task._verbose_name}' try: - with atomic(): - task.save(update_fields={'verbose_name'}) + task.save(update_fields={'verbose_name'}) except DatabaseError as e: if 'Save with update_fields did not affect any rows.' == str(e): pass @@ -200,16 +199,16 @@ def migrate_queues(): return qs.update(queue=Val(TaskQueue.NET)) +@atomic(durable=False) def schedule_media_servers_update(): - with atomic(): - # Schedule a task to update media servers - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all().iterator(): - rescan_media_server( - str(mediaserver.pk), - verbose_name=verbose_name.format(mediaserver), - ) + # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') + for mediaserver in MediaServer.objects.all().iterator(): + rescan_media_server( + str(mediaserver.pk), + verbose_name=verbose_name.format(mediaserver), + ) def cleanup_old_media(): @@ -684,18 +683,26 @@ def save_all_media_for_source(source_id): raise InvalidTaskError(_('no such source')) from e saved_later = set() - mqs = Media.objects.all().defer( - 'metadata', - 'thumb', - ).filter(source=source) - task = get_source_check_task(source_id) - refresh_qs = mqs.filter( + refresh_qs = Media.objects.all().only( + 'pk', + 'uuid', + 'key', + 'name', + ).filter( + source=source, can_download=False, skip=False, manual_skip=False, downloaded=False, metadata__isnull=False, ) + uuid_qs = Media.objects.all().only( + 'pk', + 'uuid', + ).filter( + source=source, + ).values_list('uuid', flat=True) + task = get_source_check_task(source_id) if task: task._verbose_name = remove_enclosed( task.verbose_name, '[', ']', ' ', @@ -714,7 +721,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True).iterator(chunk_size=1000), start=1): + for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: From fa9446c40e1393118d194854f2e6f1bc7429c88e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 04:32:50 -0400 Subject: [PATCH 071/454] `name` property uses `title` column --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 92a95e79..873c2876 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -687,7 +687,7 @@ def save_all_media_for_source(source_id): 'pk', 'uuid', 'key', - 'name', + 'title', # for name property ).filter( source=source, can_download=False, From 3350c0c2e28e85f8bec0630ca22562d4c3a3a148 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 04:43:10 -0400 Subject: [PATCH 072/454] fixup: missed `mqs` => `uuid_qs` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 873c2876..995aab7f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -720,7 +720,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - tvn_format = '2/{:,}' + f'/{mqs.count():,}' + tvn_format = '2/{:,}' + f'/{uuid_qs.count():,}' for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) From bb42691d623fb13b43cc399e0ca1f16a53a5aa8d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 05:05:44 -0400 Subject: [PATCH 073/454] Remove all use of the query set `iterator` function The `sqlite` driver appears to be fundamentally incompatible with the Django query set `iterator` function. It only works for the first chunk, then it fails any further database operations because the database is already locked. --- tubesync/sync/tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 995aab7f..20756b7d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -204,7 +204,7 @@ def schedule_media_servers_update(): # Schedule a task to update media servers log.info(f'Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all().iterator(): + for mediaserver in MediaServer.objects.all(): rescan_media_server( str(mediaserver.pk), verbose_name=verbose_name.format(mediaserver), @@ -213,7 +213,7 @@ def schedule_media_servers_update(): def cleanup_old_media(): with atomic(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0).iterator(chunk_size=1000): + for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): delta = timezone.now() - timedelta(days=source.days_to_keep) mqs = source.media_source.defer( 'metadata', @@ -221,7 +221,7 @@ def cleanup_old_media(): downloaded=True, download_date__lt=delta, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: log.info(f'Deleting expired media: {source} / {media} ' f'(now older than {source.days_to_keep} days / ' f'download_date before {delta})') @@ -240,7 +240,7 @@ def cleanup_removed_media(source, videos): ).filter( source=source, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: matching_source_item = [video['id'] for video in videos if video['id'] == media.key] if not matching_source_item: log.info(f'{media.name} is no longer in source, removing') @@ -710,7 +710,7 @@ def save_all_media_for_source(source_id): end=task.verbose_name.find('Check'), ) tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' - for mn, media in enumerate(refresh_qs.iterator(chunk_size=1000), start=1): + for mn, media in enumerate(refresh_qs, start=1): update_task_status(task, tvn_format.format(mn)) refresh_formats( str(media.pk), @@ -721,7 +721,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{uuid_qs.count():,}' - for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): + for mn, media_uuid in enumerate(uuid_qs, start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: @@ -788,7 +788,7 @@ def rename_all_media_for_source(source_id): source=source, downloaded=True, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: with atomic(): media.rename_files() @@ -832,7 +832,7 @@ def delete_all_media_for_source(source_id, source_name): ).filter( source=source or source_id, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): media.delete() From 8e3bfdd3c4a75eac4ea91f268e066f20dea2dc32 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 05:39:01 -0400 Subject: [PATCH 074/454] Don't sleep as long when not using the `ThreadPool` --- tubesync/sync/youtube.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 145e4c5d..61f9a489 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -205,10 +205,14 @@ def get_media_info(url, /, *, days=None, info_json=None): 'paths': paths, 'postprocessors': postprocessors, 'skip_unavailable_fragments': False, - 'sleep_interval_requests': 2 * settings.BACKGROUND_TASK_ASYNC_THREADS, + 'sleep_interval_requests': 1, 'verbose': True if settings.DEBUG else False, 'writeinfojson': True, }) + if settings.BACKGROUND_TASK_RUN_ASYNC: + opts.update({ + 'sleep_interval_requests': 2 * settings.BACKGROUND_TASK_ASYNC_THREADS, + }) if start: log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}') response = {} From 8a94efda57f2205e00f2a3fbfa9ec21c4d5de545 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 07:47:52 -0400 Subject: [PATCH 075/454] Use `django.db.reset_queries()` in workers --- tubesync/sync/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 20756b7d..f9d0f65c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -16,7 +16,7 @@ from PIL import Image from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import DatabaseError, IntegrityError +from django.db import reset_queries, DatabaseError, IntegrityError from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -254,6 +254,7 @@ def index_source_task(source_id): ''' Indexes media available from a Source object. ''' + reset_queries() cleanup_completed_tasks() # deleting expired media should happen any time an index task is requested cleanup_old_media() @@ -674,6 +675,7 @@ def save_all_media_for_source(source_id): source has its parameters changed and all media needs to be checked to see if its download status has changed. ''' + reset_queries() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: From eb206f9f4d4d46cbc9482e79c425f3aa6a1c9f84 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 10:17:09 -0400 Subject: [PATCH 076/454] Sketch for `django_queryset_generator` function --- tubesync/common/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 5894f0fc..c0e77cf4 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -222,3 +222,7 @@ def remove_enclosed(haystack, /, open='[', close=']', sep=' ', *, valid=None, st return haystack return haystack[:o] + haystack[len(n)+c:] + +def django_queryset_generator(query_set, /): + pass + From 29e94427c9ecf27911d096a0016c6990d8e870d6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 11:13:32 -0400 Subject: [PATCH 077/454] Fill in the `django_queryset_generator` function --- tubesync/common/utils.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index c0e77cf4..b74f0f76 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -1,11 +1,13 @@ import cProfile import emoji +import gc import io import os import pstats import string import time from datetime import datetime +from django.core.paginator import Paginator from urllib.parse import urlunsplit, urlencode, urlparse from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError @@ -223,6 +225,19 @@ def remove_enclosed(haystack, /, open='[', close=']', sep=' ', *, valid=None, st return haystack[:o] + haystack[len(n)+c:] -def django_queryset_generator(query_set, /): - pass +def django_queryset_generator(query_set, /, *, page_size=100): + collecting = gc.isenabled() + gc.disable() + paginator = Paginator( + query_set.values_list('pk', flat=True), + page_size, + ) + for page_num in paginator.page_range: + page = paginator.page(page_num) + for key in page.object_list: + yield query_set.filter(pk=key)[0] + gc.collect(generation=1) + gc.collect() + if collecting: + gc.enable() From 950cdbd8484120a58747ca5f7431645fcc3706ab Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 11:24:15 -0400 Subject: [PATCH 078/454] Use `django_queryset_generator` in `cleanup_old_media` --- tubesync/sync/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2ec1101a..3a9c43a6 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -27,7 +27,8 @@ from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, NoMetadataException, DownloadFailedException, ) -from common.utils import json_serial, remove_enclosed +from common.utils import ( django_queryset_generator as qa_gen, + json_serial, remove_enclosed, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, delete_file, @@ -215,7 +216,7 @@ def schedule_media_servers_update(): def cleanup_old_media(): with atomic(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): + for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)): delta = timezone.now() - timedelta(days=source.days_to_keep) mqs = source.media_source.defer( 'metadata', @@ -223,7 +224,7 @@ def cleanup_old_media(): downloaded=True, download_date__lt=delta, ) - for media in mqs: + for media in qs_gen(mqs): log.info(f'Deleting expired media: {source} / {media} ' f'(now older than {source.days_to_keep} days / ' f'download_date before {delta})') From cfceae2eb98c6ee1cbb2a5473aca66630ad21e31 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 11:26:27 -0400 Subject: [PATCH 079/454] fixup: typo --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3a9c43a6..3b8c6b86 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -27,7 +27,7 @@ from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, NoMetadataException, DownloadFailedException, ) -from common.utils import ( django_queryset_generator as qa_gen, +from common.utils import ( django_queryset_generator as qs_gen, json_serial, remove_enclosed, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer From 7c293a0444a2d9dbfc9b2492932baf55f00e5300 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 12:14:36 -0400 Subject: [PATCH 080/454] Avoid `UnorderedObjectListWarning` --- tubesync/common/utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index b74f0f76..eddcfe86 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -227,16 +227,22 @@ def remove_enclosed(haystack, /, open='[', close=']', sep=' ', *, valid=None, st def django_queryset_generator(query_set, /, *, page_size=100): collecting = gc.isenabled() + qs = query_set.values_list('pk', flat=True) + if not qs.ordered: + qs = qs.order_by('pk') + paginator = Paginator(qs, page_size) gc.disable() - paginator = Paginator( - query_set.values_list('pk', flat=True), - page_size, - ) for page_num in paginator.page_range: page = paginator.page(page_num) - for key in page.object_list: + keys = list(page.object_list) + for key in keys: yield query_set.filter(pk=key)[0] gc.collect(generation=1) + page = None + keys = list() + gc.collect() + paginator = None + qs = None gc.collect() if collecting: gc.enable() From 465abe23d022d856853761e93c82d383bd4008ea Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 13:30:26 -0400 Subject: [PATCH 081/454] Use `qs_gen` in tasks.py --- tubesync/sync/tasks.py | 81 ++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3b8c6b86..b42542fe 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -243,7 +243,7 @@ def cleanup_removed_media(source, videos): ).filter( source=source, ) - for media in mqs: + for media in qs_gen(mqs): matching_source_item = [video['id'] for video in videos if video['id'] == media.key] if not matching_source_item: log.info(f'{media.name} is no longer in source, removing') @@ -337,6 +337,7 @@ def index_source_task(source_id): update_task_status(task, None) # Cleanup of media no longer available from the source cleanup_removed_media(source, videos) + videos = video = None @background(schedule=dict(priority=0, run_at=0), queue=Val(TaskQueue.FS)) @@ -389,6 +390,7 @@ def download_source_images(source_id): file_path = source.directory_path / file_name with open(file_path, 'wb') as f: f.write(django_file.read()) + i = image_file = None if avatar != None: url = avatar @@ -404,6 +406,7 @@ def download_source_images(source_id): file_path = source.directory_path / file_name with open(file_path, 'wb') as f: f.write(django_file.read()) + i = image_file = None log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}') @@ -527,6 +530,7 @@ def download_media_thumbnail(media_id, url): ), save=True ) + i = image_file = None log.info(f'Saved thumbnail for: {media} from: {url}') return True @@ -702,7 +706,6 @@ def save_all_media_for_source(source_id): f'source exists with ID: {source_id}') raise InvalidTaskError(_('no such source')) from e - saved_later = set() refresh_qs = Media.objects.all().only( 'pk', 'uuid', @@ -716,12 +719,13 @@ def save_all_media_for_source(source_id): downloaded=False, metadata__isnull=False, ) - uuid_qs = Media.objects.all().only( + save_qs = Media.objects.all().only( 'pk', 'uuid', ).filter( source=source, - ).values_list('uuid', flat=True) + ) + saved_later = set() task = get_source_check_task(source_id) if task: task._verbose_name = remove_enclosed( @@ -730,7 +734,7 @@ def save_all_media_for_source(source_id): end=task.verbose_name.find('Check'), ) tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' - for mn, media in enumerate(refresh_qs, start=1): + for mn, media in enumerate(qs_gen(refresh_qs), start=1): update_task_status(task, tvn_format.format(mn)) refresh_formats( str(media.pk), @@ -740,18 +744,12 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - tvn_format = '2/{:,}' + f'/{uuid_qs.count():,}' - for mn, media_uuid in enumerate(uuid_qs, start=1): - if media_uuid not in saved_later: + tvn_format = '2/{:,}' + f'/{save_qs.count():,}' + for mn, media in enumerate(qs_gen(save_qs), start=1): + if media.uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) - try: - media = Media.objects.get(pk=str(media_uuid)) - except Media.DoesNotExist as e: - log.exception(str(e)) - pass - else: - with atomic(): - media.save() + with atomic(): + media.save() # Reset task.verbose_name to the saved value update_task_status(task, None) @@ -775,10 +773,12 @@ def refresh_formats(media_id): @background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.FS), remove_existing_tasks=True) def rename_media(media_id): try: - media = Media.objects.defer('metadata', 'thumb').get(pk=media_id) + media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: raise InvalidTaskError(_('no such media')) from e - media.rename_files() + else: + with atomic(): + media.rename_files() @background(schedule=dict(priority=20, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=True) @@ -802,13 +802,11 @@ def rename_all_media_for_source(source_id): ) if not create_rename_tasks: return - mqs = Media.objects.all().defer( - 'thumb', - ).filter( + mqs = Media.objects.all().filter( source=source, downloaded=True, ) - for media in mqs: + for media in qs_gen(mqs): with atomic(): media.rename_files() @@ -821,40 +819,45 @@ def wait_for_media_premiere(media_id): media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: raise InvalidTaskError(_('no such media')) from e - if media.has_metadata: - return - now = timezone.now() - if media.published < now: - media.manual_skip = False - media.skip = False - # start the download tasks - media.save() else: - media.manual_skip = True - media.title = _(f'Premieres in {hours(media.published - now)} hours') - media.save() - task = get_media_premiere_task(media_id) - if task: - update_task_status(task, f'available in {hours(media.published - now)} hours') + if media.has_metadata: + return + now = timezone.now() + if media.published < now: + media.manual_skip = False + media.skip = False + # start the download tasks after save + else: + media.manual_skip = True + media.title = _(f'Premieres in {hours(media.published - now)} hours') + task = get_media_premiere_task(media_id) + if task: + update_task_status(task, f'available in {hours(media.published - now)} hours') + with atomic(): + media.save() @background(schedule=dict(priority=1, run_at=90), queue=Val(TaskQueue.FS), remove_existing_tasks=False) def delete_all_media_for_source(source_id, source_name, source_directory): source = None + assert source_id + assert source_name + assert source_directory try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: # Task triggered but the source no longer exists, do nothing - log.error(f'Task delete_all_media_for_source(pk={source_id}) called but no ' + log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') - raise InvalidTaskError(_('no such source')) from e + #raise InvalidTaskError(_('no such source')) from e + pass # this task can run after a source was deleted mqs = Media.objects.all().defer( 'metadata', ).filter( source=source or source_id, ) with atomic(durable=True): - for media in mqs: + for media in qs_gen(mqs): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): media.delete() From 873a0167ed64991e1d2fe4ca4a7e264b1b876a9a Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 09:10:15 -0400 Subject: [PATCH 082/454] Support `use_chunked_fetch` --- tubesync/common/utils.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index eddcfe86..c4798943 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -225,23 +225,33 @@ def remove_enclosed(haystack, /, open='[', close=']', sep=' ', *, valid=None, st return haystack[:o] + haystack[len(n)+c:] -def django_queryset_generator(query_set, /, *, page_size=100): - collecting = gc.isenabled() +def django_queryset_generator(query_set, /, *, + page_size=100, + chunk_size=None, + use_chunked_fetch=False, +): qs = query_set.values_list('pk', flat=True) - if not qs.ordered: + # Avoid the `UnorderedObjectListWarning` + if not query_set.ordered: qs = qs.order_by('pk') - paginator = Paginator(qs, page_size) + collecting = gc.isenabled() gc.disable() - for page_num in paginator.page_range: - page = paginator.page(page_num) - keys = list(page.object_list) - for key in keys: + if use_chunked_fetch: + for key in qs._iterator(use_chunked_fetch, chunk_size): yield query_set.filter(pk=key)[0] + key = None gc.collect(generation=1) + key = None + else: + for page in iter(Paginator(qs, page_size)): + for key in page.object_list: + yield query_set.filter(pk=key)[0] + key = None + gc.collect(generation=1) + key = None + page = None + gc.collect() page = None - keys = list() - gc.collect() - paginator = None qs = None gc.collect() if collecting: From debac50ca3fbcd53f97f387bdce7a59764999a50 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 17:48:37 -0400 Subject: [PATCH 083/454] Fix `dumpdata` output I want a map of which functions are called, when they are called, and from which class they were defined. These things are still not clear enough for me. --- tubesync/sync/fields.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 2f479b68..11adb5e2 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -75,6 +75,7 @@ class CommaSepChoiceField(models.CharField): # Override these functions to prevent unwanted behaviors def to_python(self, value): + self.log.error(f'CommaSepChoiceField: to_python: was called with: {value!r}') return value def get_internal_type(self): @@ -154,6 +155,13 @@ class CommaSepChoiceField(models.CharField): if set(s_value) != set(value): self.log.warn(f'CommaSepChoiceField:get_prep_value: values did not match. ' f'CommaSepChoiceField({value}) versus CharField({s_value})') + return self.__class__._tuple__str__(data) + + + # extra functions not used by any parent classes + @staticmethod + def _tuple__str__(data): + value = data.selected_choices if not isinstance(value, list): return '' if data.all_choice in value: @@ -161,7 +169,6 @@ class CommaSepChoiceField(models.CharField): ordered_unique = list(dict.fromkeys(value)) return data.separator.join(ordered_unique) - # extra functions not used by any parent classes def get_all_choices(self): choice_list = list() if self.possible_choices is None: @@ -174,3 +181,6 @@ class CommaSepChoiceField(models.CharField): return choice_list + +CommaSepChoice.__str__ = CommaSepChoiceField._tuple__str__ + From cb22eb348db4e4258960abfa0edb86a43a71b41e Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 18:12:33 -0400 Subject: [PATCH 084/454] Additions from the rest container --- tubesync/sync/fields.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 11adb5e2..9f90a362 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -129,8 +129,13 @@ class CommaSepChoiceField(models.CharField): ''' Create a data structure to be used in Python code. ''' + # possibly not useful? + if isinstance(value, CommaSepChoice): + value = value.selected_choices + # normally strings if isinstance(value, str) and len(value) > 0: value = value.split(self.separator) + # empty string and None, or whatever if not isinstance(value, list): value = list() self.selected_choices = value @@ -161,6 +166,8 @@ class CommaSepChoiceField(models.CharField): # extra functions not used by any parent classes @staticmethod def _tuple__str__(data): + if not isinstance(data, CommaSepChoice): + return data value = data.selected_choices if not isinstance(value, list): return '' From 1d153954bc296b6a92a091395eae108b133f4e92 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 19:49:00 -0400 Subject: [PATCH 085/454] Try to use invalid values from `dumpdata` --- tubesync/sync/fields.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 9f90a362..e5e69498 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -75,6 +75,25 @@ class CommaSepChoiceField(models.CharField): # Override these functions to prevent unwanted behaviors def to_python(self, value): + saved_value = None + arg_was_none = True if value is None else False + if isinstance(value, CommaSepChoice): + return value.selected_choices + if isinstance(value, list) and value[0].startswith('CommaSepChoice('): + saved_value = value + value = ''.join(value) + if isinstance(value, str) and value.startswith('CommaSepChoice('): + r = value.replace('CommaSepChoice(', 'dict(', 1) + try: + o = eval(r) + except Exception: + pass + else: + return o.get('selected_choices') + if arg_was_none: + value = None + elif saved_value is not None: + value = saved_value self.log.error(f'CommaSepChoiceField: to_python: was called with: {value!r}') return value From 44a940faee4ede5913614bf682cc24a2f94ad3ef Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 19:52:44 -0400 Subject: [PATCH 086/454] fixup: empty lists should work too --- tubesync/sync/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index e5e69498..86016753 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -79,7 +79,7 @@ class CommaSepChoiceField(models.CharField): arg_was_none = True if value is None else False if isinstance(value, CommaSepChoice): return value.selected_choices - if isinstance(value, list) and value[0].startswith('CommaSepChoice('): + if isinstance(value, list) and len(value) and value[0].startswith('CommaSepChoice('): saved_value = value value = ''.join(value) if isinstance(value, str) and value.startswith('CommaSepChoice('): From 0317bc4318f1eafa26a23207e3cd0fcaa50bcb34 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 20:00:15 -0400 Subject: [PATCH 087/454] Switch the log to debug level --- tubesync/sync/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 86016753..55f9ca11 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -94,7 +94,7 @@ class CommaSepChoiceField(models.CharField): value = None elif saved_value is not None: value = saved_value - self.log.error(f'CommaSepChoiceField: to_python: was called with: {value!r}') + self.log.debug(f'CommaSepChoiceField: to_python: was called with: {value!r}') return value def get_internal_type(self): From 623a81acde8243b0291146cea80a16f0b4509bba Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 14 Apr 2025 20:04:10 -0400 Subject: [PATCH 088/454] Be consistent and explicit with the `len` check --- tubesync/sync/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 55f9ca11..452fee77 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -79,7 +79,7 @@ class CommaSepChoiceField(models.CharField): arg_was_none = True if value is None else False if isinstance(value, CommaSepChoice): return value.selected_choices - if isinstance(value, list) and len(value) and value[0].startswith('CommaSepChoice('): + if isinstance(value, list) and len(value) > 0 and value[0].startswith('CommaSepChoice('): saved_value = value value = ''.join(value) if isinstance(value, str) and value.startswith('CommaSepChoice('): From 57b7502a7bf517ad50cadf55ead9894a76ffb6a4 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 15 Apr 2025 17:47:52 -0400 Subject: [PATCH 089/454] Update README.md Link to the environment variable anchor. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3eefaef8..1bc07e07 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ and less common features: > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. > > To prevent your installation from scheduling media file renaming tasks, -> you must set `TUBESYNC_RENAME_ALL_SOURCES=False` in the environment variables. +> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables. ### 2. Index frequency From 7fd87fbb8903c63781161f7ff767e07404668640 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 15 Apr 2025 17:51:53 -0400 Subject: [PATCH 090/454] Mention `settings.py` setting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bc07e07..f506d76e 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ and less common features: > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. > > To prevent your installation from scheduling media file renaming tasks, -> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables. +> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in `settings.py`. ### 2. Index frequency From cf51901bee5b9a12d1ca96b3a177d0f483339904 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 15 Apr 2025 17:55:45 -0400 Subject: [PATCH 091/454] Link to `settings.py` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f506d76e..1f3f4408 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ and less common features: > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. > > To prevent your installation from scheduling media file renaming tasks, -> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in `settings.py`. +> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](blob/1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183). ### 2. Index frequency From 2ca1fb72e9250a786d96e888f20c5075502a25a4 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 01:25:17 -0400 Subject: [PATCH 092/454] Start using the new metadata models --- tubesync/sync/models.py | 120 ++++++++++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 34 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 075db40a..2a9e7921 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -645,7 +645,7 @@ class Media(models.Model): Source, on_delete=models.CASCADE, related_name='media_source', - help_text=_('Source the media belongs to') + help_text=_('Source the media belongs to'), ) published = models.DateTimeField( _('published'), @@ -1098,31 +1098,31 @@ class Media(models.Model): return self.metadata is not None - @atomic(durable=False) - def metadata_load(self, arg_str='{}'): + def metadata_loads(self, arg_str='{}'): data = json.loads(arg_str) or self.loaded_metadata - site = self.get_metadata_first_value('extractor_key', arg_dict=data) - epoch = self.get_metadata_first_value('epoch', arg_dict=data) - epoch_dt = self.metadata_published( epoch ) - release = self.get_metadata_first_value(('release_timestamp', 'timestamp',), arg_dict=data) - release_dt = self.metadata_published( release ) - md = self.metadata_media.get_or_create(site=site, key=self.key)[0] - md.value = data - formats = md.value.pop(self.get_metadata_field('formats'), list()) - md.retrieved = epoch_dt - md.uploaded = self.published - md.published = release_dt or self.published - md.save() - md.ingest_formats(formats) + return self.ingest_metadata(data) + + + @atomic(durable=False) + def ingest_metadata(self, data): + assert isinstance(data, dict), type(data) + md, created = self.new_metadata.get_or_create( + site=self.get_metadata_first_value('extractor_key', arg_dict=data), + key=self.key, + ) + return md.ingest_metadata(data) def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value + self.ingest_metadata(data) + epoch = self.get_metadata_first_value('epoch', arg_dict=data) + migrated = dict(migrated=True, epoch=epoch) from common.utils import json_serial - compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) + compact_json = json.dumps(migrated, separators=(',', ':'), default=json_serial) self.metadata = compact_json - self.save(update_fields={'metadata'}) + self.save() from common.logger import log log.debug(f'Saved to metadata: {self.key} / {self.uuid}: {key=}: {value}') @@ -1135,7 +1135,7 @@ class Media(models.Model): if '_reduce_data_ran_at' in data.keys(): total_seconds = data['_reduce_data_ran_at'] assert isinstance(total_seconds, int), type(total_seconds) - ran_at = self.metadata_published(total_seconds) + ran_at = self.ts_to_dt(total_seconds) if (now - ran_at) < timedelta(hours=1): return data @@ -1179,6 +1179,12 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} + data.update( + self.new_metadata.get_or_create( + site=self.get_metadata_first_value('extractor_key', arg_dict=data), + key=self.key, + )[0].with_formats + ) setattr(self, '_cached_metadata_dict', data) return data except Exception as e: @@ -1201,13 +1207,13 @@ class Media(models.Model): attempted_seconds = data.get(attempted_key) if attempted_seconds: # skip for recent unsuccessful refresh attempts also - attempted_dt = self.metadata_published(attempted_seconds) + attempted_dt = self.ts_to_dt(attempted_seconds) if (now - attempted_dt) < timedelta(seconds=self.source.index_schedule): return False # skip for recent successful formats refresh refreshed_key = 'formats_epoch' formats_seconds = data.get(refreshed_key, metadata_seconds) - metadata_dt = self.metadata_published(formats_seconds) + metadata_dt = self.ts_to_dt(formats_seconds) if (now - metadata_dt) < timedelta(seconds=self.source.index_schedule): return False @@ -1243,18 +1249,21 @@ class Media(models.Model): def metadata_title(self): return self.get_metadata_first_value(('fulltitle', 'title',), '') + def ts_to_dt(self, /, timestamp): + assert timestamp is not None + try: + timestamp_float = float(timestamp) + except Exception as e: + log.warn(f'Could not compute published from timestamp for: {self.source} / {self} with "{e}"') + pass + else: + return self.posix_epoch + timedelta(seconds=timestamp_float) + return None + def metadata_published(self, timestamp=None): if timestamp is None: timestamp = self.get_metadata_first_value('timestamp') - if timestamp is not None: - try: - timestamp_float = float(timestamp) - except Exception as e: - log.warn(f'Could not compute published from timestamp for: {self.source} / {self} with "{e}"') - pass - else: - return self.posix_epoch + timedelta(seconds=timestamp_float) - return None + return self.ts_to_dt(timestamp) @property def slugtitle(self): @@ -1736,13 +1745,14 @@ class Metadata(models.Model): default=uuid.uuid4, help_text=_('UUID of the metadata'), ) - media = models.ForeignKey( + media = models.OneToOneField( Media, # on_delete=models.DO_NOTHING, on_delete=models.CASCADE, - related_name='metadata_media', + related_name='new_metadata', help_text=_('Media the metadata belongs to'), null=False, + parent_link=True, ) site = models.CharField( _('site'), @@ -1790,13 +1800,55 @@ class Metadata(models.Model): help_text=_('JSON metadata object'), ) + @atomic(durable=False) def ingest_formats(self, formats=list(), /): for number, format in enumerate(formats, start=1): - mdf = self.metadataformat_metadata.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0] + mdf = self.metadataformat.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0] mdf.value = format mdf.save() + @property + def with_formats(self): + formats = self.metadataformat.all().order_by('number') + formats_list = [ f.value for f in formats ] + metadata = self.value.copy() + metadata.update(dict(formats=formats_list)) + return metadata + + @atomic(durable=False) + def ingest_metadata(self, data): + assert isinstance(data, dict), type(data) + self.site = self.media.get_metadata_first_value( + 'extractor_key', + arg_dict=data, + ) + self.key = self.media.key + + self.uploaded = self.media.published + self.retrieved = self.media.ts_to_dt( + self.media.get_metadata_first_value( + 'epoch', + arg_dict=data, + ) + ) + self.published = ( + self.media.ts_to_dt( + self.media.get_metadata_first_value( + ('release_timestamp', 'timestamp',), + arg_dict=data, + ) + ) or self.media.published + ) + + self.value = data.copy() # try not to have side-effects for the caller + formats_key = self.media.get_metadata_field('formats') + formats = self.value.pop(formats_key, list()) + self.save() + self.ingest_formats(formats) + + return self.with_formats + class MetadataFormat(models.Model): ''' @@ -1821,7 +1873,7 @@ class MetadataFormat(models.Model): Metadata, # on_delete=models.DO_NOTHING, on_delete=models.CASCADE, - related_name='metadataformat_metadata', + related_name='metadataformat', help_text=_('Metadata the format belongs to'), null=False, ) From 2a5bf30ab6b594dab3adb4dcd8248bb8afc857eb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 02:36:43 -0400 Subject: [PATCH 093/454] Adjust the relative link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f3f4408..e7568535 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ and less common features: > Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. > > To prevent your installation from scheduling media file renaming tasks, -> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](blob/1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183). +> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](../1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183). ### 2. Index frequency From a0e920f75c3fddff4bdbea3916bfb6d48ed6859a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 03:23:08 -0400 Subject: [PATCH 094/454] Get back to a working state for tests --- tubesync/sync/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2a9e7921..8d836d90 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1120,7 +1120,7 @@ class Media(models.Model): epoch = self.get_metadata_first_value('epoch', arg_dict=data) migrated = dict(migrated=True, epoch=epoch) from common.utils import json_serial - compact_json = json.dumps(migrated, separators=(',', ':'), default=json_serial) + compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) self.metadata = compact_json self.save() from common.logger import log @@ -1179,12 +1179,14 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} - data.update( + new_data = data.copy() + new_data.update( self.new_metadata.get_or_create( site=self.get_metadata_first_value('extractor_key', arg_dict=data), key=self.key, )[0].with_formats ) + log.debug(new_data) setattr(self, '_cached_metadata_dict', data) return data except Exception as e: From 89fcfc1c682bcc1888e74cfaae5586413185d89c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 03:31:05 -0400 Subject: [PATCH 095/454] Log the exception that is likely causing tests to fail --- tubesync/sync/models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 8d836d90..2f07e633 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1116,9 +1116,9 @@ class Media(models.Model): def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value - self.ingest_metadata(data) - epoch = self.get_metadata_first_value('epoch', arg_dict=data) - migrated = dict(migrated=True, epoch=epoch) + #self.ingest_metadata(data) + #epoch = self.get_metadata_first_value('epoch', arg_dict=data) + #migrated = dict(migrated=True, epoch=epoch) from common.utils import json_serial compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) self.metadata = compact_json @@ -1179,6 +1179,7 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} + setattr(self, '_cached_metadata_dict', data) new_data = data.copy() new_data.update( self.new_metadata.get_or_create( @@ -1187,9 +1188,9 @@ class Media(models.Model): )[0].with_formats ) log.debug(new_data) - setattr(self, '_cached_metadata_dict', data) return data except Exception as e: + log.exception(str(e)) return {} From 3e0628e4f3030e58d4ff8accc32502643a336cb7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 03:48:30 -0400 Subject: [PATCH 096/454] The field is accessed differently now --- tubesync/sync/models.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2f07e633..c9efabe0 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1116,11 +1116,11 @@ class Media(models.Model): def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value - #self.ingest_metadata(data) - #epoch = self.get_metadata_first_value('epoch', arg_dict=data) - #migrated = dict(migrated=True, epoch=epoch) + self.ingest_metadata(data) + epoch = self.get_metadata_first_value('epoch', arg_dict=data) + migrated = dict(migrated=True, epoch=epoch) from common.utils import json_serial - compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) + compact_json = json.dumps(migrated, separators=(',', ':'), default=json_serial) self.metadata = compact_json self.save() from common.logger import log @@ -1179,15 +1179,10 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} - setattr(self, '_cached_metadata_dict', data) - new_data = data.copy() - new_data.update( - self.new_metadata.get_or_create( - site=self.get_metadata_first_value('extractor_key', arg_dict=data), - key=self.key, - )[0].with_formats + data.update( + self.new_metadata.with_formats ) - log.debug(new_data) + setattr(self, '_cached_metadata_dict', data) return data except Exception as e: log.exception(str(e)) From 6062725a7915ac8ccb6f50e25cd2ca7605bc00c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 04:28:40 -0400 Subject: [PATCH 097/454] Ignore the exception when new metadata is missing --- tubesync/sync/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index c9efabe0..210db679 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1179,9 +1179,10 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} - data.update( - self.new_metadata.with_formats - ) + try: + data.update(self.new_metadata.with_formats) + except self.new_metadata.DoesNotExist: + pass setattr(self, '_cached_metadata_dict', data) return data except Exception as e: From 7c14fa3261574de45390ee9c780fc0a128258e0b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 04:36:13 -0400 Subject: [PATCH 098/454] Hard-code the class name --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 210db679..c81e5a10 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1181,7 +1181,7 @@ class Media(models.Model): return {} try: data.update(self.new_metadata.with_formats) - except self.new_metadata.DoesNotExist: + except Metadata.DoesNotExist: pass setattr(self, '_cached_metadata_dict', data) return data From d26855eb9e310023d81ec999830593c0691bb7b3 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 04:47:14 -0400 Subject: [PATCH 099/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 381dace4..50c9186b 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -24,7 +24,7 @@ class Migration(migrations.Migration): ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), - ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), + ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, parent_link=True, related_name='new_metadata', to='sync.media')), ], options={ 'verbose_name': 'Metadata about a Media item', @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), - ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), + ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat', to='sync.metadata')), ], options={ 'verbose_name': 'Format from the Metadata about a Media item', From 51261b06facca2d12125b75117a36343d307cd72 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 05:00:18 -0400 Subject: [PATCH 100/454] Use a cleaner exception --- tubesync/sync/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index c81e5a10..4858cb86 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -9,7 +9,7 @@ from pathlib import Path from xml.etree import ElementTree from django.conf import settings from django.db import models -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation from django.core.files.storage import FileSystemStorage from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import RegexValidator @@ -1179,9 +1179,10 @@ class Media(models.Model): data = json.loads(self.metadata or "{}") if not isinstance(data, dict): return {} + # if hasattr(self, 'new_metadata'): try: data.update(self.new_metadata.with_formats) - except Metadata.DoesNotExist: + except ObjectDoesNotExist: pass setattr(self, '_cached_metadata_dict', data) return data From f563619425de8704d12e1ba50fd66dbe2758f87b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 05:03:24 -0400 Subject: [PATCH 101/454] Remove unneeded white-space --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 4858cb86..3212ea42 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -9,7 +9,7 @@ from pathlib import Path from xml.etree import ElementTree from django.conf import settings from django.db import models -from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation +from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation from django.core.files.storage import FileSystemStorage from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import RegexValidator From 25ff4b64f0f4e6c2ca5ef1a3c2434599d3463157 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 05:23:02 -0400 Subject: [PATCH 102/454] Remove exception logging --- tubesync/sync/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 3212ea42..f34ea200 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1187,7 +1187,6 @@ class Media(models.Model): setattr(self, '_cached_metadata_dict', data) return data except Exception as e: - log.exception(str(e)) return {} From 029833dff617c193cc380c207bdfce466a2c592d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 05:49:13 -0400 Subject: [PATCH 103/454] Use the related_model manager --- tubesync/sync/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index f34ea200..e0fe53fa 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1106,7 +1106,9 @@ class Media(models.Model): @atomic(durable=False) def ingest_metadata(self, data): assert isinstance(data, dict), type(data) - md, created = self.new_metadata.get_or_create( + md_model = self._meta.fields_map.get('new_metadata').related_model + md, created = md_model.objects.get_or_create( + media=self, site=self.get_metadata_first_value('extractor_key', arg_dict=data), key=self.key, ) From fe47e48c430ec9ee284e2af4ca4ae98a513b08b8 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 06:43:48 -0400 Subject: [PATCH 104/454] Link to the `Media` and use a default site --- tubesync/sync/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e0fe53fa..5f03f027 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1106,10 +1106,15 @@ class Media(models.Model): @atomic(durable=False) def ingest_metadata(self, data): assert isinstance(data, dict), type(data) + site = self.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=data, + ) md_model = self._meta.fields_map.get('new_metadata').related_model md, created = md_model.objects.get_or_create( - media=self, - site=self.get_metadata_first_value('extractor_key', arg_dict=data), + media_id=self.pk, + site=site, key=self.key, ) return md.ingest_metadata(data) From 9d94729583cfeddef57c48a375cd8f84fd8bb140 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 07:09:48 -0400 Subject: [PATCH 105/454] Handle assertion errors --- tubesync/sync/models.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 5f03f027..a8b4c096 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1825,31 +1825,31 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_metadata(self, data): assert isinstance(data, dict), type(data) - self.site = self.media.get_metadata_first_value( - 'extractor_key', - arg_dict=data, - ) - self.key = self.media.key - self.uploaded = self.media.published - self.retrieved = self.media.ts_to_dt( - self.media.get_metadata_first_value( - 'epoch', - arg_dict=data, - ) - ) - self.published = ( - self.media.ts_to_dt( + try: + self.retrieved = self.media.ts_to_dt( + self.media.get_metadata_first_value( + 'epoch', + arg_dict=data, + ) + ) or self.created + except AssertionError: + self.retrieved = self.created + + try: + self.published = self.media.ts_to_dt( self.media.get_metadata_first_value( ('release_timestamp', 'timestamp',), arg_dict=data, ) ) or self.media.published - ) + except AssertionError: + self.published = self.media.published self.value = data.copy() # try not to have side-effects for the caller formats_key = self.media.get_metadata_field('formats') formats = self.value.pop(formats_key, list()) + self.uploaded = self.media.published self.save() self.ingest_formats(formats) From baedb192f4b4d97d8d2d73710bb8ccd7da0c9bcc Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 07:16:17 -0400 Subject: [PATCH 106/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 50c9186b..463c05bb 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -24,7 +24,7 @@ class Migration(migrations.Migration): ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), - ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, parent_link=True, related_name='new_metadata', to='sync.media')), + ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, parent_link=False, related_name='new_metadata', to='sync.media')), ], options={ 'verbose_name': 'Metadata about a Media item', From 24db1f072d3714f20ca13812c2d18f864b515870 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 07:17:31 -0400 Subject: [PATCH 107/454] Disable `parent_link` --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a8b4c096..d30e0ed0 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1758,7 +1758,7 @@ class Metadata(models.Model): related_name='new_metadata', help_text=_('Media the metadata belongs to'), null=False, - parent_link=True, + parent_link=False, ) site = models.CharField( _('site'), From 47a3220c42d5ef4a38df9241e0acf34312ab34e5 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 07:57:38 -0400 Subject: [PATCH 108/454] Invalidate the cached metadata --- tubesync/sync/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index d30e0ed0..ed489ee7 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1117,6 +1117,7 @@ class Media(models.Model): site=site, key=self.key, ) + setattr(self, '_cached_metadata_dict', None) return md.ingest_metadata(data) From 4777b851137775c80c385fffc79609507e5fab0e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 15:17:33 -0400 Subject: [PATCH 109/454] Indentation --- tubesync/sync/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index ed489ee7..fb79ebbd 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -21,12 +21,12 @@ from common.logger import log from common.errors import NoFormatException from common.utils import clean_filename, clean_emoji from .youtube import (get_media_info as get_youtube_media_info, - download_media as download_youtube_media, - get_channel_image_info as get_youtube_channel_image_info) + download_media as download_youtube_media, + get_channel_image_info as get_youtube_channel_image_info) from .utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote) from .matching import (get_best_combined_format, get_best_audio_format, - get_best_video_format) + get_best_video_format) from .fields import CommaSepChoiceField from .choices import (Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, From 44b92643eb80e9cf4630bed0bbbe779e6726abbc Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 15:40:45 -0400 Subject: [PATCH 110/454] Index on site & key --- tubesync/sync/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index fb79ebbd..a2b72c8f 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1765,6 +1765,7 @@ class Metadata(models.Model): _('site'), max_length=256, blank=True, + db_index=True, null=False, default='Youtube', help_text=_('Site from which the metadata was retrieved'), @@ -1773,6 +1774,7 @@ class Metadata(models.Model): _('key'), max_length=256, blank=True, + db_index=True, null=False, default='', help_text=_('Media identifier at the site from which the metadata was retrieved'), @@ -1791,11 +1793,13 @@ class Metadata(models.Model): ) uploaded = models.DateTimeField( _('uploaded'), + db_index=True, null=True, help_text=_('Date and time the media was uploaded'), ) published = models.DateTimeField( _('published'), + db_index=True, null=True, help_text=_('Date and time the media was published'), ) @@ -1888,6 +1892,7 @@ class MetadataFormat(models.Model): _('site'), max_length=256, blank=True, + db_index=True, null=False, default='Youtube', help_text=_('Site from which the format is available'), @@ -1896,6 +1901,7 @@ class MetadataFormat(models.Model): _('key'), max_length=256, blank=True, + db_index=True, null=False, default='', help_text=_('Media identifier at the site for which this format is available'), From 2ef653a29a73461388ce9ef0951db1a64c0e130d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Apr 2025 15:56:44 -0400 Subject: [PATCH 111/454] Update 0031_metadata_metadataformat.py --- .../migrations/0031_metadata_metadataformat.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 463c05bb..00bcec08 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -17,14 +17,14 @@ class Migration(migrations.Migration): name='Metadata', fields=[ ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the metadata', primary_key=True, serialize=False, verbose_name='uuid')), - ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site')), - ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key')), + ('site', models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key')), ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was created', verbose_name='created')), ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), - ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), - ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), + ('uploaded', models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), + ('published', models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), - ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, parent_link=False, related_name='new_metadata', to='sync.media')), + ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='new_metadata', to='sync.media')), ], options={ 'verbose_name': 'Metadata about a Media item', @@ -36,8 +36,8 @@ class Migration(migrations.Migration): name='MetadataFormat', fields=[ ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the format', primary_key=True, serialize=False, verbose_name='uuid')), - ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), - ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), + ('site', models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), From d60a159af7405548249ebf856d5d3c882291b7d2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 02:48:46 -0400 Subject: [PATCH 112/454] Add delay for SQLite to allow for more interleaving --- tubesync/sync/tasks.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index cb2b0a70..2c654749 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -7,6 +7,8 @@ import os import json import math +import random +import time import uuid from io import BytesIO from hashlib import sha1 @@ -17,7 +19,8 @@ from PIL import Image from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import connection, reset_queries, DatabaseError, IntegrityError +from django.db import reset_queries, DatabaseError, IntegrityError +from django.db.connection import vendor as db_vendor from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -203,10 +206,12 @@ def migrate_queues(): def save_model(instance): - if 'sqlite' == connection.vendor: + if 'sqlite' == db_vendor: # a transaction here causes too many # database is locked errors + # with atomic(): instance.save() + time.sleep(random.expovariate(1.5)) else: with atomic(): instance.save() @@ -752,6 +757,14 @@ def save_all_media_for_source(source_id): ) saved_later.add(media.uuid) + # Keep out of the way of the index task! + # SQLite will be locked for a while if we start + # a large source, which reschedules a more costly task. + if 'sqlite' == db_vendor: + index_task = get_source_index_task(source_id) + if index_task and index_task.locked_by_pid_running(): + raise Exception(_('Indexing not completed')) + # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{save_qs.count():,}' From 09969afa6e42c5ee37c9cad46cac8e3b443e48e7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 02:53:46 -0400 Subject: [PATCH 113/454] Import and set the variable --- tubesync/sync/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2c654749..3c461e38 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -16,11 +16,11 @@ from pathlib import Path from datetime import datetime, timedelta from shutil import copyfile, rmtree from PIL import Image +from django import db from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.db import reset_queries, DatabaseError, IntegrityError -from django.db.connection import vendor as db_vendor from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -38,6 +38,8 @@ from .utils import ( get_remote_image, resize_image_to_height, delete_file, write_text_file, filter_response, ) from .youtube import YouTubeError +db_vendor = db.connection.vendor + def get_hash(task_name, pk): ''' From 4bf975bf92d1c95dde265a66b7492c415326e03b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:03:00 -0400 Subject: [PATCH 114/454] Move the SQLite case to the end of the function --- tubesync/sync/tasks.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3c461e38..17696295 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -208,15 +208,16 @@ def migrate_queues(): def save_model(instance): - if 'sqlite' == db_vendor: - # a transaction here causes too many - # database is locked errors - # with atomic(): - instance.save() - time.sleep(random.expovariate(1.5)) - else: + if 'sqlite' != db_vendor: with atomic(): instance.save() + return + + # work around for SQLite and its many + # "database is locked" errors + with atomic(): + instance.save() + time.sleep(random.expovariate(1.5)) @atomic(durable=False) From 3b5c7766347c64e761551ba5238cea73ea11aa41 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:06:17 -0400 Subject: [PATCH 115/454] Do not interfere with the exception handling --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 17696295..d19b9f70 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -332,7 +332,7 @@ def index_source_task(source_id): if published_dt is not None: media.published = published_dt try: - save_model(media) + media.save() except IntegrityError as e: log.error(f'Index media failed: {source} / {media} with "{e}"') else: From b1e7f0eed2cffbad6bea28b2d7183071ca67879f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:11:14 -0400 Subject: [PATCH 116/454] Be explicit about not using durable with atomic --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d19b9f70..34a723d2 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -209,13 +209,13 @@ def migrate_queues(): def save_model(instance): if 'sqlite' != db_vendor: - with atomic(): + with atomic(durable=False): instance.save() return # work around for SQLite and its many # "database is locked" errors - with atomic(): + with atomic(durable=False): instance.save() time.sleep(random.expovariate(1.5)) From 035eeea54cfc97613ff8e8e97b55d6f0dc57dde4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:24:05 -0400 Subject: [PATCH 117/454] Add `SQLITE_DELAY_FLOAT` setting --- tubesync/tubesync/local_settings.py.container | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 4f386b66..bcd70330 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -59,6 +59,12 @@ else: } DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"' + # the argument to random.expovariate(), + # a larger value means less delay + # with too little delay, you may see + # more "database is locked" errors + SQLITE_DELAY_FLOAT = 1.5 + DEFAULT_THREADS = 1 BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True) From 134231fda7d5033614b12f5cd0bfe260e4482816 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:27:17 -0400 Subject: [PATCH 118/454] Use the new setting --- tubesync/sync/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 34a723d2..87cf15a7 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -217,7 +217,8 @@ def save_model(instance): # "database is locked" errors with atomic(durable=False): instance.save() - time.sleep(random.expovariate(1.5)) + arg = getattr(settings, 'SQLITE_DELAY_FLOAT', 1.5) + time.sleep(random.expovariate(arg)) @atomic(durable=False) From 142e93500f8f3ad1dea3909b56eafed6b5ae1740 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 03:31:10 -0400 Subject: [PATCH 119/454] Use the db import for reset_queries --- tubesync/sync/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 87cf15a7..2b055ce5 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -20,7 +20,7 @@ from django import db from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import reset_queries, DatabaseError, IntegrityError +from django.db import DatabaseError, IntegrityError from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -276,7 +276,7 @@ def index_source_task(source_id): ''' Indexes media available from a Source object. ''' - reset_queries() + db.reset_queries() cleanup_completed_tasks() # deleting expired media should happen any time an index task is requested cleanup_old_media() @@ -716,7 +716,7 @@ def save_all_media_for_source(source_id): source has its parameters changed and all media needs to be checked to see if its download status has changed. ''' - reset_queries() + db.reset_queries() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: From a1495d27f0d211406cd045adfe8822bede0dd336 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 12:45:49 -0400 Subject: [PATCH 120/454] Update local_settings.py.container --- tubesync/tubesync/local_settings.py.container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index bcd70330..8b61692b 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -63,7 +63,7 @@ else: # a larger value means less delay # with too little delay, you may see # more "database is locked" errors - SQLITE_DELAY_FLOAT = 1.5 + SQLITE_DELAY_FLOAT = 5 DEFAULT_THREADS = 1 From 624406eb27f0114753c0b52fb1487b332dc14f1d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 14:41:21 -0400 Subject: [PATCH 121/454] Add /etc/apt/apt.conf.d/docker-disable-pkgcache --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index d3169884..5af1acab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ FROM debian:${DEBIAN_VERSION} AS tubesync-base ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" \ + APT_KEEP_ARCHIVES=1 \ HOME="/root" \ LANGUAGE="en_US.UTF-8" \ LANG="en_US.UTF-8" \ @@ -39,6 +40,11 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va rm -f /var/cache/apt/*cache.bin ; \ # Update from the network and keep cache rm -f /etc/apt/apt.conf.d/docker-clean ; \ + # Do not generate more /var/cache/apt/*cache.bin files + # hopefully soon, this will be included in Debian images + printf -- >| /etc/apt/apt.conf.d/docker-disable-pkgcache \ + 'Dir::Cache::%spkgcache "";\n' '' src ; \ + chmod a+r /etc/apt/apt.conf.d/docker-disable-pkgcache ; \ set -x && \ apt-get update && \ # Install locales From a6fea907200a0e408991be4b556a2635569463f7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 15:28:20 -0400 Subject: [PATCH 122/454] Remove a raw value For now, `file_ext` has the same value as, `CHECKSUM_ALGORITHM`, but I used another variable to make changing it easier. --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5af1acab..3cbeda4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -227,15 +227,16 @@ RUN set -eu ; \ unset -v arg1 ; \ } ; \ \ + file_ext="${CHECKSUM_ALGORITHM}" ; \ apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \ mkdir -v /verified ; \ cd /downloaded ; \ - for f in *.sha256 ; \ + for f in *."${file_ext}" ; \ do \ "${CHECKSUM_ALGORITHM}sum" --check --warn --strict "${f}" || exit ; \ - ln -v "${f%.sha256}" /verified/ || exit ; \ + ln -v "${f%.${file_ext}}" /verified/ || exit ; \ done ; \ - unset -v f ; \ + unset -v f file_ext ; \ \ S6_ARCH="$(decide_arch "${TARGETARCH}")" ; \ set -x ; \ From 8fe43170a4d2691ae40612391463a7adfefffc87 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 15:32:14 -0400 Subject: [PATCH 123/454] Debian on `arm` is `armhf` Not that we are likely to use this, but I figured this out, and it costs nothing to leave a case here. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3cbeda4c..d68efe3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -221,7 +221,7 @@ RUN set -eu ; \ case "${arg1}" in \ (amd64) printf -- 'x86_64' ;; \ (arm64) printf -- 'aarch64' ;; \ - (armv7l) printf -- 'arm' ;; \ + (arm|armv7l) printf -- 'armhf' ;; \ (*) printf -- '%s' "${arg1}" ;; \ esac ; \ unset -v arg1 ; \ From 083e29a1891b640c299e06da1fe2b196913458da Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 15:37:55 -0400 Subject: [PATCH 124/454] Save a step for test branches --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c5c46c5b..6204097a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,10 +7,11 @@ on: workflow_dispatch: push: branches: - - main + - 'main' + - 'test-*' pull_request: branches: - - main + - 'main' types: - opened - reopened From 970e8c9a603a3afd04f88c0566e16fab6d9d5a5d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 15:43:36 -0400 Subject: [PATCH 125/454] Adjustments for the `perl-base` upgrade --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6204097a..9edc7225 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -136,14 +136,15 @@ jobs: push: false tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:dive - name: Analysis with `dive` + continue-on-error: false run: | docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ 'ghcr.io/wagoodman/dive' \ 'ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:dive' \ --ci \ - --highestUserWastedPercent '0.03' \ - --highestWastedBytes '10M' + --highestUserWastedPercent '0.05' \ + --highestWastedBytes '50M' - name: Build and push id: build-push timeout-minutes: 60 From 8ec35292b95a8a4c76f6d3b61043e58acc1f3bed Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 20:24:01 -0400 Subject: [PATCH 126/454] Don't modify the original metadata yet I've lost some metadata during testing because of deleting the new `Metadata` instances without restoring the original metadata JSON. --- tubesync/sync/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a2b72c8f..be62ab6c 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1125,10 +1125,10 @@ class Media(models.Model): data = self.loaded_metadata data[key] = value self.ingest_metadata(data) - epoch = self.get_metadata_first_value('epoch', arg_dict=data) - migrated = dict(migrated=True, epoch=epoch) + #epoch = self.get_metadata_first_value('epoch', arg_dict=data) + #migrated = dict(migrated=True, epoch=epoch) from common.utils import json_serial - compact_json = json.dumps(migrated, separators=(',', ':'), default=json_serial) + compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) self.metadata = compact_json self.save() from common.logger import log From 45dba712c5bb9ca294454d53bb09764e65a5c669 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 21:23:33 -0400 Subject: [PATCH 127/454] Remove extra white-space --- tubesync/sync/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index be62ab6c..f303f123 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -853,7 +853,7 @@ class Media(models.Model): ''' fetch the first key with a value from metadata ''' - + if arg_dict is None: arg_dict = self.loaded_metadata assert isinstance(arg_dict, dict), type(arg_dict) @@ -932,7 +932,7 @@ class Media(models.Model): return str(fmt.get('id')) return False return False - + def get_display_format(self, format_str): ''' Returns a tuple used in the format component of the output filename. This @@ -1230,7 +1230,7 @@ class Media(models.Model): metadata = self.index_metadata() if self.skip: return False - + response = metadata if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) From c46d044c51c0d545ba1d87059313e6ef766143bf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 22:02:37 -0400 Subject: [PATCH 128/454] Add new models to admin.py --- tubesync/sync/admin.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/admin.py b/tubesync/sync/admin.py index 1e445b7d..41a99999 100644 --- a/tubesync/sync/admin.py +++ b/tubesync/sync/admin.py @@ -1,5 +1,11 @@ from django.contrib import admin -from .models import Source, Media, MediaServer +from .models import ( + Source, + Media, + Metadata, + MetadataFormat, + MediaServer +) @admin.register(Source) @@ -21,6 +27,24 @@ class MediaAdmin(admin.ModelAdmin): search_fields = ('uuid', 'source__key', 'key') +@admin.register(Metadata) +class MetadataAdmin(admin.ModelAdmin): + + ordering = ('-created',) + list_display = ('uuid', 'media', 'site', 'key', 'created', 'retrieved', 'uploaded', 'published') + readonly_fields = ('uuid', 'created') + search_fields = ('uuid', 'site', 'key') + + +@admin.register(MetadataFormat) +class MetadataFormatAdmin(admin.ModelAdmin): + + ordering = ('site', 'key', 'number') + list_display = ('uuid', 'metadata', 'site', 'key', 'code') + readonly_fields = ('uuid', 'metadata') + search_fields = ('uuid', 'site', 'key', 'code') + + @admin.register(MediaServer) class MediaServerAdmin(admin.ModelAdmin): From 934e4c9fcb758af81098f89b732691140dd1b43f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 22:57:56 -0400 Subject: [PATCH 129/454] Adjust admin displays --- tubesync/sync/admin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/admin.py b/tubesync/sync/admin.py index 41a99999..9589c37b 100644 --- a/tubesync/sync/admin.py +++ b/tubesync/sync/admin.py @@ -30,19 +30,19 @@ class MediaAdmin(admin.ModelAdmin): @admin.register(Metadata) class MetadataAdmin(admin.ModelAdmin): - ordering = ('-created',) - list_display = ('uuid', 'media', 'site', 'key', 'created', 'retrieved', 'uploaded', 'published') - readonly_fields = ('uuid', 'created') - search_fields = ('uuid', 'site', 'key') + ordering = ('-retrieved', '-created', '-uploaded') + list_display = ('uuid', 'key', 'retrieved', 'uploaded', 'created', 'site') + readonly_fields = ('uuid', 'created', 'retrieved') + search_fields = ('uuid', 'media', 'key') @admin.register(MetadataFormat) class MetadataFormatAdmin(admin.ModelAdmin): ordering = ('site', 'key', 'number') - list_display = ('uuid', 'metadata', 'site', 'key', 'code') - readonly_fields = ('uuid', 'metadata') - search_fields = ('uuid', 'site', 'key', 'code') + list_display = ('uuid', 'key', 'site', 'code', 'number', 'metadata') + readonly_fields = ('uuid', 'metadata', 'site', 'key', 'number') + search_fields = ('uuid', 'key') @admin.register(MediaServer) From 5516cad86b786bf92c6a1e0474638e1540c70559 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Apr 2025 23:37:09 -0400 Subject: [PATCH 130/454] Search by `Media.uuid` on metadata tables --- tubesync/sync/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/admin.py b/tubesync/sync/admin.py index 9589c37b..f4ecfbc0 100644 --- a/tubesync/sync/admin.py +++ b/tubesync/sync/admin.py @@ -33,7 +33,7 @@ class MetadataAdmin(admin.ModelAdmin): ordering = ('-retrieved', '-created', '-uploaded') list_display = ('uuid', 'key', 'retrieved', 'uploaded', 'created', 'site') readonly_fields = ('uuid', 'created', 'retrieved') - search_fields = ('uuid', 'media', 'key') + search_fields = ('uuid', 'media__uuid', 'key') @admin.register(MetadataFormat) @@ -42,7 +42,7 @@ class MetadataFormatAdmin(admin.ModelAdmin): ordering = ('site', 'key', 'number') list_display = ('uuid', 'key', 'site', 'code', 'number', 'metadata') readonly_fields = ('uuid', 'metadata', 'site', 'key', 'number') - search_fields = ('uuid', 'key') + search_fields = ('uuid', 'metadata__uuid', 'metadata__media__uuid', 'key') @admin.register(MediaServer) From 60ffc4cdc893f8bcfd6a071f3d22f7300aaa7a0a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Apr 2025 19:58:29 -0400 Subject: [PATCH 131/454] Use `Media.has_metadata` in `Media.save` function --- tubesync/sync/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 075db40a..c8d837a1 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -830,14 +830,21 @@ class Media(models.Model): update_fields = {'media_file', 'skip'}.union(update_fields) # Trigger an update of derived fields from metadata - if update_fields is None or 'metadata' in update_fields: + update_md = ( + self.created and + self.has_metadata and + ( + update_fields is None or + 'metadata' in update_fields + ) + ) + if update_md: setattr(self, '_cached_metadata_dict', None) - if self.metadata: self.title = self.metadata_title[:200] self.duration = self.metadata_duration - if update_fields is not None and "metadata" in update_fields: - # If only some fields are being updated, make sure we update title and duration if metadata changes - update_fields = {"title", "duration"}.union(update_fields) + if update_fields is not None: + # If only some fields are being updated, make sure we update title and duration if metadata changes + update_fields = {"title", "duration"}.union(update_fields) super().save( force_insert=force_insert, From 1c9ef93ec99be730e02d409264bc446bb1f55b40 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Apr 2025 21:03:23 -0400 Subject: [PATCH 132/454] Remove `created` condition --- tubesync/sync/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index c8d837a1..b41c30f7 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -831,7 +831,6 @@ class Media(models.Model): # Trigger an update of derived fields from metadata update_md = ( - self.created and self.has_metadata and ( update_fields is None or From eee233962b9bd87bbacffd502a3b7814f57b008b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 11:47:12 -0400 Subject: [PATCH 133/454] Remove extra white-space --- tubesync/sync/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index b41c30f7..8fd8eb5f 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -859,7 +859,7 @@ class Media(models.Model): ''' fetch the first key with a value from metadata ''' - + if arg_dict is None: arg_dict = self.loaded_metadata assert isinstance(arg_dict, dict), type(arg_dict) @@ -938,7 +938,7 @@ class Media(models.Model): return str(fmt.get('id')) return False return False - + def get_display_format(self, format_str): ''' Returns a tuple used in the format component of the output filename. This @@ -1223,7 +1223,7 @@ class Media(models.Model): metadata = self.index_metadata() if self.skip: return False - + response = metadata if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) From e03c98d0a4a3655773a91cd1918cd6b37ac2386c Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 13:07:03 -0400 Subject: [PATCH 134/454] Squash old `sync` migrations --- tubesync/sync/migrations/0001_initial.py | 94 ---- ...quashed_0030_alter_source_source_vcodec.py | 406 ++++++++++++++++++ .../migrations/0002_auto_20201213_0817.py | 20 - .../migrations/0003_source_copy_thumbnails.py | 18 - .../migrations/0004_source_media_format.py | 18 - .../migrations/0005_auto_20201219_0312.py | 18 - .../sync/migrations/0006_source_write_nfo.py | 18 - .../migrations/0007_auto_20201219_0645.py | 18 - .../migrations/0008_source_download_cap.py | 18 - .../migrations/0009_auto_20210218_0442.py | 30 -- .../migrations/0010_auto_20210924_0554.py | 30 -- .../migrations/0011_auto_20220201_1654.py | 21 - .../0012_alter_media_downloaded_format.py | 18 - .../migrations/0013_fix_elative_media_file.py | 25 -- .../migrations/0014_alter_media_media_file.py | 21 - .../migrations/0015_auto_20230213_0603.py | 23 - .../migrations/0016_auto_20230214_2052.py | 34 -- ...17_alter_source_sponsorblock_categories.py | 19 - .../sync/migrations/0018_source_subtitles.py | 27 -- .../0019_add_delete_removed_media.py | 17 - .../migrations/0020_auto_20231024_1825.py | 29 -- .../0021_source_copy_channel_images.py | 18 - .../0022_add_delete_files_on_disk.py | 17 - .../migrations/0023_media_duration_filter.py | 62 --- .../migrations/0024_auto_20240717_1535.py | 28 -- .../migrations/0025_add_video_type_support.py | 20 - .../migrations/0026_alter_source_sub_langs.py | 19 - ...27_alter_source_sponsorblock_categories.py | 19 - .../0028_alter_source_source_resolution.py | 17 - .../0029_alter_mediaserver_fields.py | 33 -- .../0030_alter_source_source_vcodec.py | 18 - 31 files changed, 406 insertions(+), 767 deletions(-) delete mode 100644 tubesync/sync/migrations/0001_initial.py create mode 100644 tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py delete mode 100644 tubesync/sync/migrations/0002_auto_20201213_0817.py delete mode 100644 tubesync/sync/migrations/0003_source_copy_thumbnails.py delete mode 100644 tubesync/sync/migrations/0004_source_media_format.py delete mode 100644 tubesync/sync/migrations/0005_auto_20201219_0312.py delete mode 100644 tubesync/sync/migrations/0006_source_write_nfo.py delete mode 100644 tubesync/sync/migrations/0007_auto_20201219_0645.py delete mode 100644 tubesync/sync/migrations/0008_source_download_cap.py delete mode 100644 tubesync/sync/migrations/0009_auto_20210218_0442.py delete mode 100644 tubesync/sync/migrations/0010_auto_20210924_0554.py delete mode 100644 tubesync/sync/migrations/0011_auto_20220201_1654.py delete mode 100644 tubesync/sync/migrations/0012_alter_media_downloaded_format.py delete mode 100644 tubesync/sync/migrations/0013_fix_elative_media_file.py delete mode 100644 tubesync/sync/migrations/0014_alter_media_media_file.py delete mode 100644 tubesync/sync/migrations/0015_auto_20230213_0603.py delete mode 100644 tubesync/sync/migrations/0016_auto_20230214_2052.py delete mode 100644 tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py delete mode 100644 tubesync/sync/migrations/0018_source_subtitles.py delete mode 100644 tubesync/sync/migrations/0019_add_delete_removed_media.py delete mode 100644 tubesync/sync/migrations/0020_auto_20231024_1825.py delete mode 100644 tubesync/sync/migrations/0021_source_copy_channel_images.py delete mode 100644 tubesync/sync/migrations/0022_add_delete_files_on_disk.py delete mode 100644 tubesync/sync/migrations/0023_media_duration_filter.py delete mode 100644 tubesync/sync/migrations/0024_auto_20240717_1535.py delete mode 100644 tubesync/sync/migrations/0025_add_video_type_support.py delete mode 100644 tubesync/sync/migrations/0026_alter_source_sub_langs.py delete mode 100644 tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py delete mode 100644 tubesync/sync/migrations/0028_alter_source_source_resolution.py delete mode 100644 tubesync/sync/migrations/0029_alter_mediaserver_fields.py delete mode 100644 tubesync/sync/migrations/0030_alter_source_source_vcodec.py diff --git a/tubesync/sync/migrations/0001_initial.py b/tubesync/sync/migrations/0001_initial.py deleted file mode 100644 index aa267a9a..00000000 --- a/tubesync/sync/migrations/0001_initial.py +++ /dev/null @@ -1,94 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-12 09:13 - -import django.core.files.storage -from django.db import migrations, models -import django.db.models.deletion -import sync.models -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Source', - fields=[ - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created')), - ('last_crawl', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl')), - ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type')), - ('key', models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key')), - ('name', models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name')), - ('directory', models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory')), - ('index_schedule', models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours')], db_index=True, default=21600, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule')), - ('delete_old_media', models.BooleanField(default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media')), - ('days_to_keep', models.PositiveSmallIntegerField(default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep')), - ('source_resolution', models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution')), - ('source_vcodec', models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec')), - ('source_acodec', models.CharField(choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec')), - ('prefer_60fps', models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps')), - ('prefer_hdr', models.BooleanField(default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr')), - ('fallback', models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback')), - ('has_failed', models.BooleanField(default=False, help_text='Source has failed to index media', verbose_name='has failed')), - ], - options={ - 'verbose_name': 'Source', - 'verbose_name_plural': 'Sources', - }, - ), - migrations.CreateModel( - name='MediaServer', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('server_type', models.CharField(choices=[('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type')), - ('host', models.CharField(db_index=True, help_text='Hostname or IP address of the media server', max_length=200, verbose_name='host')), - ('port', models.PositiveIntegerField(db_index=True, help_text='Port number of the media server', verbose_name='port')), - ('use_https', models.BooleanField(default=True, help_text='Connect to the media server over HTTPS', verbose_name='use https')), - ('verify_https', models.BooleanField(default=False, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https')), - ('options', models.TextField(blank=True, help_text='JSON encoded options for the media server', null=True, verbose_name='options')), - ], - options={ - 'verbose_name': 'Media Server', - 'verbose_name_plural': 'Media Servers', - 'unique_together': {('host', 'port')}, - }, - ), - migrations.CreateModel( - name='Media', - fields=[ - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created')), - ('published', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published')), - ('key', models.CharField(db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key')), - ('thumb', models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width')), - ('thumb_width', models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width')), - ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height')), - ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')), - ('can_download', models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download')), - ('media_file', models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file')), - ('skip', models.BooleanField(db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip')), - ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')), - ('download_date', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date')), - ('downloaded_format', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded format')), - ('downloaded_height', models.PositiveIntegerField(blank=True, help_text='Height in pixels of the downloaded media', null=True, verbose_name='downloaded height')), - ('downloaded_width', models.PositiveIntegerField(blank=True, help_text='Width in pixels of the downloaded media', null=True, verbose_name='downloaded width')), - ('downloaded_audio_codec', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec')), - ('downloaded_video_codec', models.CharField(blank=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec')), - ('downloaded_container', models.CharField(blank=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format')), - ('downloaded_fps', models.PositiveSmallIntegerField(blank=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps')), - ('downloaded_hdr', models.BooleanField(default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr')), - ('downloaded_filesize', models.PositiveBigIntegerField(blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize')), - ('source', models.ForeignKey(help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source')), - ], - options={ - 'verbose_name': 'Media', - 'verbose_name_plural': 'Media', - 'unique_together': {('source', 'key')}, - }, - ), - ] diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py new file mode 100644 index 00000000..48bc23b7 --- /dev/null +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -0,0 +1,406 @@ +# Manually adjusted based on the generated file. +# Generated by Django 5.1.8 on 2025-04-10 15:29 + +import django.core.files.storage +import django.core.validators +import django.db.models.deletion +import sync.fields +import sync.models +import uuid +from django.db import migrations, models + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# sync.migrations.0013_fix_elative_media_file +from django.conf import settings +from pathlib import Path + +def fix_media_file(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + download_dir = str(settings.DOWNLOAD_ROOT) + download_dir_path = Path(download_dir) + for media in Media.objects.filter(downloaded=True): + if media.media_file.path.startswith(download_dir): + media_path = Path(media.media_file.path) + relative_path = media_path.relative_to(download_dir_path) + media.media_file.name = str(relative_path) + media.save() + +# Function above has been copied/modified and RunPython operations adjusted. + +def media_file_location(): + return str(settings.DOWNLOAD_ROOT)) + +# Used the above function for storage location. + +class Migration(migrations.Migration): + + replaces = [ + ('sync', '0001_initial'), + ('sync', '0002_auto_20201213_0817'), + ('sync', '0003_source_copy_thumbnails'), + ('sync', '0004_source_media_format'), + ('sync', '0005_auto_20201219_0312'), + ('sync', '0006_source_write_nfo'), + ('sync', '0007_auto_20201219_0645'), + ('sync', '0008_source_download_cap'), + ('sync', '0009_auto_20210218_0442'), + ('sync', '0010_auto_20210924_0554'), + ('sync', '0011_auto_20220201_1654'), + ('sync', '0012_alter_media_downloaded_format'), + ('sync', '0013_fix_elative_media_file'), + ('sync', '0014_alter_media_media_file'), + ('sync', '0015_auto_20230213_0603'), + ('sync', '0016_auto_20230214_2052'), + ('sync', '0017_alter_source_sponsorblock_categories'), + ('sync', '0018_source_subtitles'), + ('sync', '0019_add_delete_removed_media'), + ('sync', '0020_auto_20231024_1825'), + ('sync', '0021_source_copy_channel_images'), + ('sync', '0022_add_delete_files_on_disk'), + ('sync', '0023_media_duration_filter'), + ('sync', '0024_auto_20240717_1535'), + ('sync', '0025_add_video_type_support'), + ('sync', '0026_alter_source_sub_langs'), + ('sync', '0027_alter_source_sponsorblock_categories'), + ('sync', '0028_alter_source_source_resolution'), + ('sync', '0029_alter_mediaserver_fields'), + ('sync', '0030_alter_source_source_vcodec'), + ] + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Source', + fields=[ + ('uuid', models.UUIDField( + default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid', + )), + ('created', models.DateTimeField( + auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created', + )), + ('last_crawl', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl', + )), + ('source_type', models.CharField( + choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type', + )), + ('key', models.CharField( + db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key', + )), + ('name', models.CharField( + db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name', + )), + ('directory', models.CharField( + db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory', + )), + ('index_schedule', models.IntegerField( + choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule', + )), + ('delete_old_media', models.BooleanField( + default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media', + )), + ('days_to_keep', models.PositiveSmallIntegerField( + default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep', + )), + ('source_resolution', models.CharField( + choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution', + )), + ('source_vcodec', models.CharField( + choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec', + )), + ('source_acodec', models.CharField( + choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec', + )), + ('prefer_60fps', models.BooleanField( + default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps', + )), + ('prefer_hdr', models.BooleanField( + default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr', + )), + ('fallback', models.CharField( + choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback', + )), + ('has_failed', models.BooleanField( + default=False, help_text='Source has failed to index media', verbose_name='has failed', + )), + ('copy_thumbnails', models.BooleanField( + default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails', + )), + ('media_format', models.CharField( + default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format', + )), + ('write_nfo', models.BooleanField( + default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo', + )), + ('download_cap', models.IntegerField( + choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap', + )), + ('download_media', models.BooleanField( + default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media', + )), + ('write_json', models.BooleanField( + default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json', + )), + ], + options={ + 'verbose_name': 'Source', + 'verbose_name_plural': 'Sources', + }, + ), + migrations.CreateModel( + name='MediaServer', + fields=[ + ('id', models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID', + )), + ('server_type', models.CharField( + choices=[('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type', + )), + ('host', models.CharField( + db_index=True, help_text='Hostname or IP address of the media server', max_length=200, verbose_name='host', + )), + ('port', models.PositiveIntegerField( + db_index=True, help_text='Port number of the media server', verbose_name='port', + )), + ('use_https', models.BooleanField( + default=True, help_text='Connect to the media server over HTTPS', verbose_name='use https', + )), + ('verify_https', models.BooleanField( + default=False, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https', + )), + ('options', models.TextField( + blank=True, help_text='JSON encoded options for the media server', null=True, verbose_name='options', + )), + ], + options={ + 'verbose_name': 'Media Server', + 'verbose_name_plural': 'Media Servers', + 'unique_together': {('host', 'port')}, + }, + ), + migrations.CreateModel( + name='Media', + fields=[ + ('uuid', models.UUIDField( + default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid', + )), + ('created', models.DateTimeField( + auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created', + )), + ('published', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published', + )), + ('key', models.CharField( + db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key', + )), + ('thumb', models.ImageField( + blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width', + )), + ('thumb_width', models.PositiveSmallIntegerField( + blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width', + )), + ('thumb_height', models.PositiveSmallIntegerField( + blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height', + )), + ('metadata', models.TextField( + blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata', + )), + ('can_download', models.BooleanField( + db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download', + )), + ('media_file', models.FileField( + blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location(), upload_to=sync.models.get_media_file_path, verbose_name='media file', + )), + ('skip', models.BooleanField( + db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip', + )), + ('downloaded', models.BooleanField( + db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded', + )), + ('download_date', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date', + )), + ('downloaded_format', models.CharField( + blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format', + )), + ('downloaded_height', models.PositiveIntegerField( + blank=True, help_text='Height in pixels of the downloaded media', null=True, verbose_name='downloaded height', + )), + ('downloaded_width', models.PositiveIntegerField( + blank=True, help_text='Width in pixels of the downloaded media', null=True, verbose_name='downloaded width', + )), + ('downloaded_audio_codec', models.CharField( + blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec', + )), + ('downloaded_video_codec', models.CharField( + blank=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec', + )), + ('downloaded_container', models.CharField( + blank=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format', + )), + ('downloaded_fps', models.PositiveSmallIntegerField( + blank=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps', + )), + ('downloaded_hdr', models.BooleanField( + default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr', + )), + ('downloaded_filesize', models.PositiveBigIntegerField( + blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize', + )), + ('source', models.ForeignKey( + help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source', + )), + ], + options={ + 'verbose_name': 'Media', + 'verbose_name_plural': 'Media', + 'unique_together': {('source', 'key')}, + }, + ), + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location(), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AlterField( + model_name='media', + name='skip', + field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), + ), + migrations.AddField( + model_name='source', + name='embed_metadata', + field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), + ), + migrations.AddField( + model_name='source', + name='embed_thumbnail', + field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), + ), + migrations.AddField( + model_name='source', + name='enable_sponsorblock', + field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), + ), + migrations.AddField( + model_name='source', + name='write_subtitles', + field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), + ), + migrations.AddField( + model_name='source', + name='delete_removed_media', + field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), + ), + migrations.AddField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), + ), + migrations.AddField( + model_name='source', + name='copy_channel_images', + field=models.BooleanField(default=False, help_text='Copy channel banner and avatar. These may be detected and used by some media servers', verbose_name='copy channel images'), + ), + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + migrations.AddField( + model_name='media', + name='duration', + field=models.PositiveIntegerField(blank=True, help_text='Duration of media in seconds', null=True, verbose_name='duration'), + ), + migrations.AddField( + model_name='source', + name='filter_seconds', + field=models.PositiveIntegerField(blank=True, help_text='Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering', null=True, verbose_name='filter seconds'), + ), + migrations.AddField( + model_name='source', + name='filter_seconds_min', + field=models.BooleanField(choices=[(True, 'Minimum Length'), (False, 'Maximum Length')], default=True, help_text='When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video greater than maximum) video duration', verbose_name='filter seconds min/max'), + ), + migrations.AddField( + model_name='source', + name='filter_text_invert', + field=models.BooleanField(default=False, help_text='Invert filter string regex match, skip any matching titles when selected', verbose_name='invert filter text matching'), + ), + migrations.AddField( + model_name='media', + name='manual_skip', + field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\'t be downloaded', verbose_name='manual_skip'), + ), + migrations.AddField( + model_name='media', + name='title', + field=models.CharField(blank=True, default='', help_text='Video title', max_length=200, verbose_name='title'), + ), + migrations.AddField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=200, verbose_name='filter string'), + ), + migrations.AddField( + model_name='source', + name='index_videos', + field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + migrations.AddField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), + ), + migrations.AddField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField(all_choice='all', all_label='(All Categories)', allow_all=True, default='all', help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', max_length=128, possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''), + ), + migrations.AlterField( + model_name='source', + name='source_resolution', + field=models.CharField(choices=[('audio', 'Audio only'), ('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), + ), + migrations.AlterField( + model_name='mediaserver', + name='options', + field=models.TextField(help_text='JSON encoded options for the media server', null=True, verbose_name='options'), + ), + migrations.AlterField( + model_name='mediaserver', + name='server_type', + field=models.CharField(choices=[('j', 'Jellyfin'), ('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type'), + ), + migrations.AlterField( + model_name='mediaserver', + name='use_https', + field=models.BooleanField(default=False, help_text='Connect to the media server over HTTPS', verbose_name='use https'), + ), + migrations.AlterField( + model_name='mediaserver', + name='verify_https', + field=models.BooleanField(default=True, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https'), + ), + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9'), ('AV1', 'AV1')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + migrations.RunPython( + code=fix_media_file, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/tubesync/sync/migrations/0002_auto_20201213_0817.py b/tubesync/sync/migrations/0002_auto_20201213_0817.py deleted file mode 100644 index ab2f71e3..00000000 --- a/tubesync/sync/migrations/0002_auto_20201213_0817.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-13 08:17 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - ] diff --git a/tubesync/sync/migrations/0003_source_copy_thumbnails.py b/tubesync/sync/migrations/0003_source_copy_thumbnails.py deleted file mode 100644 index 7bdbfdbc..00000000 --- a/tubesync/sync/migrations/0003_source_copy_thumbnails.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-18 01:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0002_auto_20201213_0817'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='copy_thumbnails', - field=models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails'), - ), - ] diff --git a/tubesync/sync/migrations/0004_source_media_format.py b/tubesync/sync/migrations/0004_source_media_format.py deleted file mode 100644 index f79a7036..00000000 --- a/tubesync/sync/migrations/0004_source_media_format.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-18 01:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0003_source_copy_thumbnails'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0005_auto_20201219_0312.py b/tubesync/sync/migrations/0005_auto_20201219_0312.py deleted file mode 100644 index 11ee8e27..00000000 --- a/tubesync/sync/migrations/0005_auto_20201219_0312.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 03:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0004_source_media_format'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_type', - field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'), - ), - ] diff --git a/tubesync/sync/migrations/0006_source_write_nfo.py b/tubesync/sync/migrations/0006_source_write_nfo.py deleted file mode 100644 index d5fe9365..00000000 --- a/tubesync/sync/migrations/0006_source_write_nfo.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 03:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0005_auto_20201219_0312'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_nfo', - field=models.BooleanField(default=False, help_text='Write an NFO file with the media, these may be detected and used by some media servers', verbose_name='write nfo'), - ), - ] diff --git a/tubesync/sync/migrations/0007_auto_20201219_0645.py b/tubesync/sync/migrations/0007_auto_20201219_0645.py deleted file mode 100644 index c757d679..00000000 --- a/tubesync/sync/migrations/0007_auto_20201219_0645.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 06:45 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0006_source_write_nfo'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='write_nfo', - field=models.BooleanField(default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo'), - ), - ] diff --git a/tubesync/sync/migrations/0008_source_download_cap.py b/tubesync/sync/migrations/0008_source_download_cap.py deleted file mode 100644 index 4b3ec19f..00000000 --- a/tubesync/sync/migrations/0008_source_download_cap.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 06:59 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0007_auto_20201219_0645'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='download_cap', - field=models.IntegerField(choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap'), - ), - ] diff --git a/tubesync/sync/migrations/0009_auto_20210218_0442.py b/tubesync/sync/migrations/0009_auto_20210218_0442.py deleted file mode 100644 index 45b94500..00000000 --- a/tubesync/sync/migrations/0009_auto_20210218_0442.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 3.1.6 on 2021-02-18 04:42 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0008_source_download_cap'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='download_media', - field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'), - ), - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - migrations.AlterField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0010_auto_20210924_0554.py b/tubesync/sync/migrations/0010_auto_20210924_0554.py deleted file mode 100644 index 00160610..00000000 --- a/tubesync/sync/migrations/0010_auto_20210924_0554.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 3.2.7 on 2021-09-24 05:54 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0009_auto_20210218_0442'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - migrations.AlterField( - model_name='source', - name='index_schedule', - field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'), - ), - migrations.AlterField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py deleted file mode 100644 index 96d9f4a7..00000000 --- a/tubesync/sync/migrations/0011_auto_20220201_1654.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 3.2.11 on 2022-02-01 16:54 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0010_auto_20210924_0554'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_json', - field=models.BooleanField( - default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'), - ), - ] diff --git a/tubesync/sync/migrations/0012_alter_media_downloaded_format.py b/tubesync/sync/migrations/0012_alter_media_downloaded_format.py deleted file mode 100644 index 3e733efb..00000000 --- a/tubesync/sync/migrations/0012_alter_media_downloaded_format.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.12 on 2022-04-06 06:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0011_auto_20220201_1654'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='downloaded_format', - field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), - ), - ] diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py deleted file mode 100644 index c9eee22e..00000000 --- a/tubesync/sync/migrations/0013_fix_elative_media_file.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 3.2.12 on 2022-04-06 06:19 - -from django.conf import settings -from django.db import migrations, models - - -def fix_media_file(apps, schema_editor): - Media = apps.get_model('sync', 'Media') - for media in Media.objects.filter(downloaded=True): - download_dir = str(settings.DOWNLOAD_ROOT) - - if media.media_file.name.startswith(download_dir): - media.media_file.name = media.media_file.name[len(download_dir) + 1:] - media.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0012_alter_media_downloaded_format'), - ] - - operations = [ - migrations.RunPython(fix_media_file) - ] diff --git a/tubesync/sync/migrations/0014_alter_media_media_file.py b/tubesync/sync/migrations/0014_alter_media_media_file.py deleted file mode 100644 index 530c8e81..00000000 --- a/tubesync/sync/migrations/0014_alter_media_media_file.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 3.2.15 on 2022-12-28 20:33 - -import django.core.files.storage -from django.conf import settings -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0013_fix_elative_media_file'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - ] diff --git a/tubesync/sync/migrations/0015_auto_20230213_0603.py b/tubesync/sync/migrations/0015_auto_20230213_0603.py deleted file mode 100644 index 54592f9d..00000000 --- a/tubesync/sync/migrations/0015_auto_20230213_0603.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.2.17 on 2023-02-13 06:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0014_alter_media_media_file'), - ] - - operations = [ - migrations.AddField( - model_name='media', - name='manual_skip', - field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'), - ), - migrations.AlterField( - model_name='media', - name='skip', - field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), - ), - ] diff --git a/tubesync/sync/migrations/0016_auto_20230214_2052.py b/tubesync/sync/migrations/0016_auto_20230214_2052.py deleted file mode 100644 index ffba1952..00000000 --- a/tubesync/sync/migrations/0016_auto_20230214_2052.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 3.2.18 on 2023-02-14 20:52 - -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0015_auto_20230213_0603'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='embed_metadata', - field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), - ), - migrations.AddField( - model_name='source', - name='embed_thumbnail', - field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), - ), - migrations.AddField( - model_name='source', - name='enable_sponsorblock', - field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), - ), - migrations.AddField( - model_name='source', - name='sponsorblock_categories', - field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), - ), - ] diff --git a/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py deleted file mode 100644 index cc9d9578..00000000 --- a/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.18 on 2023-02-20 02:23 - -from django.db import migrations -import sync.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0016_auto_20230214_2052'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''), - ), - ] diff --git a/tubesync/sync/migrations/0018_source_subtitles.py b/tubesync/sync/migrations/0018_source_subtitles.py deleted file mode 100644 index c526b994..00000000 --- a/tubesync/sync/migrations/0018_source_subtitles.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0017_alter_source_sponsorblock_categories'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_subtitles', - field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), - ), - migrations.AddField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'), - ), - migrations.AddField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30), - ), - ] diff --git a/tubesync/sync/migrations/0019_add_delete_removed_media.py b/tubesync/sync/migrations/0019_add_delete_removed_media.py deleted file mode 100644 index 0762be87..00000000 --- a/tubesync/sync/migrations/0019_add_delete_removed_media.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0018_source_subtitles'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='delete_removed_media', - field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), - ), - ] diff --git a/tubesync/sync/migrations/0020_auto_20231024_1825.py b/tubesync/sync/migrations/0020_auto_20231024_1825.py deleted file mode 100644 index 295339a8..00000000 --- a/tubesync/sync/migrations/0020_auto_20231024_1825.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.22 on 2023-10-24 17:25 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0019_add_delete_removed_media'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), - ), - migrations.AlterField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), - ), - migrations.AlterField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), - ), - ] diff --git a/tubesync/sync/migrations/0021_source_copy_channel_images.py b/tubesync/sync/migrations/0021_source_copy_channel_images.py deleted file mode 100644 index 5d568925..00000000 --- a/tubesync/sync/migrations/0021_source_copy_channel_images.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by nothing. Done manually by InterN0te on 2023-12-10 16:36 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0020_auto_20231024_1825'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='copy_channel_images', - field=models.BooleanField(default=False, help_text='Copy channel banner and avatar. These may be detected and used by some media servers', verbose_name='copy channel images'), - ), - ] diff --git a/tubesync/sync/migrations/0022_add_delete_files_on_disk.py b/tubesync/sync/migrations/0022_add_delete_files_on_disk.py deleted file mode 100644 index 959f3d8b..00000000 --- a/tubesync/sync/migrations/0022_add_delete_files_on_disk.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0021_source_copy_channel_images'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='delete_files_on_disk', - field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), - ), - ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0023_media_duration_filter.py b/tubesync/sync/migrations/0023_media_duration_filter.py deleted file mode 100644 index 558cc0ef..00000000 --- a/tubesync/sync/migrations/0023_media_duration_filter.py +++ /dev/null @@ -1,62 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("sync", "0022_add_delete_files_on_disk"), - ] - - operations = [ - migrations.AddField( - model_name="media", - name="title", - field=models.CharField( - verbose_name="title", - max_length=100, - blank=True, - null=False, - default="", - help_text="Video title", - ), - ), - migrations.AddField( - model_name="media", - name="duration", - field=models.PositiveIntegerField( - verbose_name="duration", - blank=True, - null=True, - help_text="Duration of media in seconds", - ), - ), - migrations.AddField( - model_name="source", - name="filter_seconds", - field=models.PositiveIntegerField( - verbose_name="filter seconds", - blank=True, - null=True, - help_text="Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering", - ), - ), - migrations.AddField( - model_name="source", - name="filter_seconds_min", - field=models.BooleanField( - verbose_name="filter seconds min/max", - choices=[(True, "Minimum Length"), (False, "Maximum Length")], - default=True, - help_text="When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (" - "video greater than maximum) video duration", - ), - ), - migrations.AddField( - model_name="source", - name="filter_text_invert", - field=models.BooleanField( - verbose_name="invert filter text matching", - default=False, - help_text="Invert filter string regex match, skip any matching titles when selected", - ), - ), - ] diff --git a/tubesync/sync/migrations/0024_auto_20240717_1535.py b/tubesync/sync/migrations/0024_auto_20240717_1535.py deleted file mode 100644 index c2d20d35..00000000 --- a/tubesync/sync/migrations/0024_auto_20240717_1535.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 3.2.25 on 2024-07-17 15:35 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0023_media_duration_filter'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='manual_skip', - field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\'t be downloaded', verbose_name='manual_skip'), - ), - migrations.AlterField( - model_name='media', - name='title', - field=models.CharField(blank=True, default='', help_text='Video title', max_length=200, verbose_name='title'), - ), - migrations.AlterField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=200, verbose_name='filter string'), - ), - ] diff --git a/tubesync/sync/migrations/0025_add_video_type_support.py b/tubesync/sync/migrations/0025_add_video_type_support.py deleted file mode 100644 index e1a901c0..00000000 --- a/tubesync/sync/migrations/0025_add_video_type_support.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0024_auto_20240717_1535'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='index_videos', - field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), - ), - migrations.AddField( - model_name='source', - name='index_streams', - field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), - ), - ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0026_alter_source_sub_langs.py b/tubesync/sync/migrations/0026_alter_source_sub_langs.py deleted file mode 100644 index 937c3a14..00000000 --- a/tubesync/sync/migrations/0026_alter_source_sub_langs.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.25 on 2024-12-11 12:43 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0025_add_video_type_support'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), - ), - ] diff --git a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py deleted file mode 100644 index c81b8e72..00000000 --- a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.25 on 2025-01-29 06:14 - -from django.db import migrations -import sync.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0026_alter_source_sub_langs'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(all_choice='all', all_label='(All Categories)', allow_all=True, default='all', help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', max_length=128, possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''), - ), - ] diff --git a/tubesync/sync/migrations/0028_alter_source_source_resolution.py b/tubesync/sync/migrations/0028_alter_source_source_resolution.py deleted file mode 100644 index e72f7307..00000000 --- a/tubesync/sync/migrations/0028_alter_source_source_resolution.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.2.25 on 2025-02-12 18:31 - -from django.db import migrations, models - -class Migration(migrations.Migration): - dependencies = [ - ('sync', '0027_alter_source_sponsorblock_categories'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_resolution', - field=models.CharField(choices=[('audio', 'Audio only'), ('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), - ), - ] - diff --git a/tubesync/sync/migrations/0029_alter_mediaserver_fields.py b/tubesync/sync/migrations/0029_alter_mediaserver_fields.py deleted file mode 100644 index b7363430..00000000 --- a/tubesync/sync/migrations/0029_alter_mediaserver_fields.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 3.2.25 on 2025-02-22 03:52 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0028_alter_source_source_resolution'), - ] - - operations = [ - migrations.AlterField( - model_name='mediaserver', - name='options', - field=models.TextField(help_text='JSON encoded options for the media server', null=True, verbose_name='options'), - ), - migrations.AlterField( - model_name='mediaserver', - name='server_type', - field=models.CharField(choices=[('j', 'Jellyfin'), ('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type'), - ), - migrations.AlterField( - model_name='mediaserver', - name='use_https', - field=models.BooleanField(default=False, help_text='Connect to the media server over HTTPS', verbose_name='use https'), - ), - migrations.AlterField( - model_name='mediaserver', - name='verify_https', - field=models.BooleanField(default=True, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https'), - ), - ] diff --git a/tubesync/sync/migrations/0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0030_alter_source_source_vcodec.py deleted file mode 100644 index 2b4f3618..00000000 --- a/tubesync/sync/migrations/0030_alter_source_source_vcodec.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.8 on 2025-04-07 18:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0029_alter_mediaserver_fields'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_vcodec', - field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9'), ('AV1', 'AV1')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), - ), - ] From 725a7d2d616a30a5a1be700086b6901cef6852f5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 13:27:18 -0400 Subject: [PATCH 135/454] fixup: remove an extra `)` --- .../migrations/0001_squashed_0030_alter_source_source_vcodec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 48bc23b7..ed41856c 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -31,7 +31,7 @@ def fix_media_file(apps, schema_editor): # Function above has been copied/modified and RunPython operations adjusted. def media_file_location(): - return str(settings.DOWNLOAD_ROOT)) + return str(settings.DOWNLOAD_ROOT) # Used the above function for storage location. From df456824731bc14cca8898faa2a6b4981889dd82 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 13:41:54 -0400 Subject: [PATCH 136/454] fixup: add missing `)` --- .../0001_squashed_0030_alter_source_source_vcodec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index ed41856c..ac4e87eb 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -216,7 +216,7 @@ class Migration(migrations.Migration): db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download', )), ('media_file', models.FileField( - blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location(), upload_to=sync.models.get_media_file_path, verbose_name='media file', + blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file', )), ('skip', models.BooleanField( db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip', @@ -267,7 +267,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='media', name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location(), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file'), ), migrations.AlterField( model_name='media', From 6de4f55105b3fac66817a36ba5f1120a4b50d7bf Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 13:49:30 -0400 Subject: [PATCH 137/454] Squash old migrations for `sync` --- tubesync/sync/migrations/0001_initial.py | 94 ---- ...quashed_0030_alter_source_source_vcodec.py | 406 ++++++++++++++++++ .../migrations/0002_auto_20201213_0817.py | 20 - .../migrations/0003_source_copy_thumbnails.py | 18 - .../migrations/0004_source_media_format.py | 18 - .../migrations/0005_auto_20201219_0312.py | 18 - .../sync/migrations/0006_source_write_nfo.py | 18 - .../migrations/0007_auto_20201219_0645.py | 18 - .../migrations/0008_source_download_cap.py | 18 - .../migrations/0009_auto_20210218_0442.py | 30 -- .../migrations/0010_auto_20210924_0554.py | 30 -- .../migrations/0011_auto_20220201_1654.py | 21 - .../0012_alter_media_downloaded_format.py | 18 - .../migrations/0013_fix_elative_media_file.py | 25 -- .../migrations/0014_alter_media_media_file.py | 21 - .../migrations/0015_auto_20230213_0603.py | 23 - .../migrations/0016_auto_20230214_2052.py | 34 -- ...17_alter_source_sponsorblock_categories.py | 19 - .../sync/migrations/0018_source_subtitles.py | 27 -- .../0019_add_delete_removed_media.py | 17 - .../migrations/0020_auto_20231024_1825.py | 29 -- .../0021_source_copy_channel_images.py | 18 - .../0022_add_delete_files_on_disk.py | 17 - .../migrations/0023_media_duration_filter.py | 62 --- .../migrations/0024_auto_20240717_1535.py | 28 -- .../migrations/0025_add_video_type_support.py | 20 - .../migrations/0026_alter_source_sub_langs.py | 19 - ...27_alter_source_sponsorblock_categories.py | 19 - .../0028_alter_source_source_resolution.py | 17 - .../0029_alter_mediaserver_fields.py | 33 -- .../0030_alter_source_source_vcodec.py | 18 - 31 files changed, 406 insertions(+), 767 deletions(-) delete mode 100644 tubesync/sync/migrations/0001_initial.py create mode 100644 tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py delete mode 100644 tubesync/sync/migrations/0002_auto_20201213_0817.py delete mode 100644 tubesync/sync/migrations/0003_source_copy_thumbnails.py delete mode 100644 tubesync/sync/migrations/0004_source_media_format.py delete mode 100644 tubesync/sync/migrations/0005_auto_20201219_0312.py delete mode 100644 tubesync/sync/migrations/0006_source_write_nfo.py delete mode 100644 tubesync/sync/migrations/0007_auto_20201219_0645.py delete mode 100644 tubesync/sync/migrations/0008_source_download_cap.py delete mode 100644 tubesync/sync/migrations/0009_auto_20210218_0442.py delete mode 100644 tubesync/sync/migrations/0010_auto_20210924_0554.py delete mode 100644 tubesync/sync/migrations/0011_auto_20220201_1654.py delete mode 100644 tubesync/sync/migrations/0012_alter_media_downloaded_format.py delete mode 100644 tubesync/sync/migrations/0013_fix_elative_media_file.py delete mode 100644 tubesync/sync/migrations/0014_alter_media_media_file.py delete mode 100644 tubesync/sync/migrations/0015_auto_20230213_0603.py delete mode 100644 tubesync/sync/migrations/0016_auto_20230214_2052.py delete mode 100644 tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py delete mode 100644 tubesync/sync/migrations/0018_source_subtitles.py delete mode 100644 tubesync/sync/migrations/0019_add_delete_removed_media.py delete mode 100644 tubesync/sync/migrations/0020_auto_20231024_1825.py delete mode 100644 tubesync/sync/migrations/0021_source_copy_channel_images.py delete mode 100644 tubesync/sync/migrations/0022_add_delete_files_on_disk.py delete mode 100644 tubesync/sync/migrations/0023_media_duration_filter.py delete mode 100644 tubesync/sync/migrations/0024_auto_20240717_1535.py delete mode 100644 tubesync/sync/migrations/0025_add_video_type_support.py delete mode 100644 tubesync/sync/migrations/0026_alter_source_sub_langs.py delete mode 100644 tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py delete mode 100644 tubesync/sync/migrations/0028_alter_source_source_resolution.py delete mode 100644 tubesync/sync/migrations/0029_alter_mediaserver_fields.py delete mode 100644 tubesync/sync/migrations/0030_alter_source_source_vcodec.py diff --git a/tubesync/sync/migrations/0001_initial.py b/tubesync/sync/migrations/0001_initial.py deleted file mode 100644 index aa267a9a..00000000 --- a/tubesync/sync/migrations/0001_initial.py +++ /dev/null @@ -1,94 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-12 09:13 - -import django.core.files.storage -from django.db import migrations, models -import django.db.models.deletion -import sync.models -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Source', - fields=[ - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created')), - ('last_crawl', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl')), - ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type')), - ('key', models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key')), - ('name', models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name')), - ('directory', models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory')), - ('index_schedule', models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours')], db_index=True, default=21600, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule')), - ('delete_old_media', models.BooleanField(default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media')), - ('days_to_keep', models.PositiveSmallIntegerField(default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep')), - ('source_resolution', models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution')), - ('source_vcodec', models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec')), - ('source_acodec', models.CharField(choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec')), - ('prefer_60fps', models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps')), - ('prefer_hdr', models.BooleanField(default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr')), - ('fallback', models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback')), - ('has_failed', models.BooleanField(default=False, help_text='Source has failed to index media', verbose_name='has failed')), - ], - options={ - 'verbose_name': 'Source', - 'verbose_name_plural': 'Sources', - }, - ), - migrations.CreateModel( - name='MediaServer', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('server_type', models.CharField(choices=[('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type')), - ('host', models.CharField(db_index=True, help_text='Hostname or IP address of the media server', max_length=200, verbose_name='host')), - ('port', models.PositiveIntegerField(db_index=True, help_text='Port number of the media server', verbose_name='port')), - ('use_https', models.BooleanField(default=True, help_text='Connect to the media server over HTTPS', verbose_name='use https')), - ('verify_https', models.BooleanField(default=False, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https')), - ('options', models.TextField(blank=True, help_text='JSON encoded options for the media server', null=True, verbose_name='options')), - ], - options={ - 'verbose_name': 'Media Server', - 'verbose_name_plural': 'Media Servers', - 'unique_together': {('host', 'port')}, - }, - ), - migrations.CreateModel( - name='Media', - fields=[ - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created')), - ('published', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published')), - ('key', models.CharField(db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key')), - ('thumb', models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width')), - ('thumb_width', models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width')), - ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height')), - ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')), - ('can_download', models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download')), - ('media_file', models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file')), - ('skip', models.BooleanField(db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip')), - ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')), - ('download_date', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date')), - ('downloaded_format', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded format')), - ('downloaded_height', models.PositiveIntegerField(blank=True, help_text='Height in pixels of the downloaded media', null=True, verbose_name='downloaded height')), - ('downloaded_width', models.PositiveIntegerField(blank=True, help_text='Width in pixels of the downloaded media', null=True, verbose_name='downloaded width')), - ('downloaded_audio_codec', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec')), - ('downloaded_video_codec', models.CharField(blank=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec')), - ('downloaded_container', models.CharField(blank=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format')), - ('downloaded_fps', models.PositiveSmallIntegerField(blank=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps')), - ('downloaded_hdr', models.BooleanField(default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr')), - ('downloaded_filesize', models.PositiveBigIntegerField(blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize')), - ('source', models.ForeignKey(help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source')), - ], - options={ - 'verbose_name': 'Media', - 'verbose_name_plural': 'Media', - 'unique_together': {('source', 'key')}, - }, - ), - ] diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py new file mode 100644 index 00000000..ac4e87eb --- /dev/null +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -0,0 +1,406 @@ +# Manually adjusted based on the generated file. +# Generated by Django 5.1.8 on 2025-04-10 15:29 + +import django.core.files.storage +import django.core.validators +import django.db.models.deletion +import sync.fields +import sync.models +import uuid +from django.db import migrations, models + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# sync.migrations.0013_fix_elative_media_file +from django.conf import settings +from pathlib import Path + +def fix_media_file(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + download_dir = str(settings.DOWNLOAD_ROOT) + download_dir_path = Path(download_dir) + for media in Media.objects.filter(downloaded=True): + if media.media_file.path.startswith(download_dir): + media_path = Path(media.media_file.path) + relative_path = media_path.relative_to(download_dir_path) + media.media_file.name = str(relative_path) + media.save() + +# Function above has been copied/modified and RunPython operations adjusted. + +def media_file_location(): + return str(settings.DOWNLOAD_ROOT) + +# Used the above function for storage location. + +class Migration(migrations.Migration): + + replaces = [ + ('sync', '0001_initial'), + ('sync', '0002_auto_20201213_0817'), + ('sync', '0003_source_copy_thumbnails'), + ('sync', '0004_source_media_format'), + ('sync', '0005_auto_20201219_0312'), + ('sync', '0006_source_write_nfo'), + ('sync', '0007_auto_20201219_0645'), + ('sync', '0008_source_download_cap'), + ('sync', '0009_auto_20210218_0442'), + ('sync', '0010_auto_20210924_0554'), + ('sync', '0011_auto_20220201_1654'), + ('sync', '0012_alter_media_downloaded_format'), + ('sync', '0013_fix_elative_media_file'), + ('sync', '0014_alter_media_media_file'), + ('sync', '0015_auto_20230213_0603'), + ('sync', '0016_auto_20230214_2052'), + ('sync', '0017_alter_source_sponsorblock_categories'), + ('sync', '0018_source_subtitles'), + ('sync', '0019_add_delete_removed_media'), + ('sync', '0020_auto_20231024_1825'), + ('sync', '0021_source_copy_channel_images'), + ('sync', '0022_add_delete_files_on_disk'), + ('sync', '0023_media_duration_filter'), + ('sync', '0024_auto_20240717_1535'), + ('sync', '0025_add_video_type_support'), + ('sync', '0026_alter_source_sub_langs'), + ('sync', '0027_alter_source_sponsorblock_categories'), + ('sync', '0028_alter_source_source_resolution'), + ('sync', '0029_alter_mediaserver_fields'), + ('sync', '0030_alter_source_source_vcodec'), + ] + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Source', + fields=[ + ('uuid', models.UUIDField( + default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid', + )), + ('created', models.DateTimeField( + auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created', + )), + ('last_crawl', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl', + )), + ('source_type', models.CharField( + choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type', + )), + ('key', models.CharField( + db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key', + )), + ('name', models.CharField( + db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name', + )), + ('directory', models.CharField( + db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory', + )), + ('index_schedule', models.IntegerField( + choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule', + )), + ('delete_old_media', models.BooleanField( + default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media', + )), + ('days_to_keep', models.PositiveSmallIntegerField( + default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep', + )), + ('source_resolution', models.CharField( + choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution', + )), + ('source_vcodec', models.CharField( + choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec', + )), + ('source_acodec', models.CharField( + choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec', + )), + ('prefer_60fps', models.BooleanField( + default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps', + )), + ('prefer_hdr', models.BooleanField( + default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr', + )), + ('fallback', models.CharField( + choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback', + )), + ('has_failed', models.BooleanField( + default=False, help_text='Source has failed to index media', verbose_name='has failed', + )), + ('copy_thumbnails', models.BooleanField( + default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails', + )), + ('media_format', models.CharField( + default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format', + )), + ('write_nfo', models.BooleanField( + default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo', + )), + ('download_cap', models.IntegerField( + choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap', + )), + ('download_media', models.BooleanField( + default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media', + )), + ('write_json', models.BooleanField( + default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json', + )), + ], + options={ + 'verbose_name': 'Source', + 'verbose_name_plural': 'Sources', + }, + ), + migrations.CreateModel( + name='MediaServer', + fields=[ + ('id', models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID', + )), + ('server_type', models.CharField( + choices=[('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type', + )), + ('host', models.CharField( + db_index=True, help_text='Hostname or IP address of the media server', max_length=200, verbose_name='host', + )), + ('port', models.PositiveIntegerField( + db_index=True, help_text='Port number of the media server', verbose_name='port', + )), + ('use_https', models.BooleanField( + default=True, help_text='Connect to the media server over HTTPS', verbose_name='use https', + )), + ('verify_https', models.BooleanField( + default=False, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https', + )), + ('options', models.TextField( + blank=True, help_text='JSON encoded options for the media server', null=True, verbose_name='options', + )), + ], + options={ + 'verbose_name': 'Media Server', + 'verbose_name_plural': 'Media Servers', + 'unique_together': {('host', 'port')}, + }, + ), + migrations.CreateModel( + name='Media', + fields=[ + ('uuid', models.UUIDField( + default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid', + )), + ('created', models.DateTimeField( + auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created', + )), + ('published', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published', + )), + ('key', models.CharField( + db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key', + )), + ('thumb', models.ImageField( + blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width', + )), + ('thumb_width', models.PositiveSmallIntegerField( + blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width', + )), + ('thumb_height', models.PositiveSmallIntegerField( + blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height', + )), + ('metadata', models.TextField( + blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata', + )), + ('can_download', models.BooleanField( + db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download', + )), + ('media_file', models.FileField( + blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file', + )), + ('skip', models.BooleanField( + db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip', + )), + ('downloaded', models.BooleanField( + db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded', + )), + ('download_date', models.DateTimeField( + blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date', + )), + ('downloaded_format', models.CharField( + blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format', + )), + ('downloaded_height', models.PositiveIntegerField( + blank=True, help_text='Height in pixels of the downloaded media', null=True, verbose_name='downloaded height', + )), + ('downloaded_width', models.PositiveIntegerField( + blank=True, help_text='Width in pixels of the downloaded media', null=True, verbose_name='downloaded width', + )), + ('downloaded_audio_codec', models.CharField( + blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec', + )), + ('downloaded_video_codec', models.CharField( + blank=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec', + )), + ('downloaded_container', models.CharField( + blank=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format', + )), + ('downloaded_fps', models.PositiveSmallIntegerField( + blank=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps', + )), + ('downloaded_hdr', models.BooleanField( + default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr', + )), + ('downloaded_filesize', models.PositiveBigIntegerField( + blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize', + )), + ('source', models.ForeignKey( + help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source', + )), + ], + options={ + 'verbose_name': 'Media', + 'verbose_name_plural': 'Media', + 'unique_together': {('source', 'key')}, + }, + ), + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AlterField( + model_name='media', + name='skip', + field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), + ), + migrations.AddField( + model_name='source', + name='embed_metadata', + field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), + ), + migrations.AddField( + model_name='source', + name='embed_thumbnail', + field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), + ), + migrations.AddField( + model_name='source', + name='enable_sponsorblock', + field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), + ), + migrations.AddField( + model_name='source', + name='write_subtitles', + field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), + ), + migrations.AddField( + model_name='source', + name='delete_removed_media', + field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), + ), + migrations.AddField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), + ), + migrations.AddField( + model_name='source', + name='copy_channel_images', + field=models.BooleanField(default=False, help_text='Copy channel banner and avatar. These may be detected and used by some media servers', verbose_name='copy channel images'), + ), + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + migrations.AddField( + model_name='media', + name='duration', + field=models.PositiveIntegerField(blank=True, help_text='Duration of media in seconds', null=True, verbose_name='duration'), + ), + migrations.AddField( + model_name='source', + name='filter_seconds', + field=models.PositiveIntegerField(blank=True, help_text='Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering', null=True, verbose_name='filter seconds'), + ), + migrations.AddField( + model_name='source', + name='filter_seconds_min', + field=models.BooleanField(choices=[(True, 'Minimum Length'), (False, 'Maximum Length')], default=True, help_text='When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video greater than maximum) video duration', verbose_name='filter seconds min/max'), + ), + migrations.AddField( + model_name='source', + name='filter_text_invert', + field=models.BooleanField(default=False, help_text='Invert filter string regex match, skip any matching titles when selected', verbose_name='invert filter text matching'), + ), + migrations.AddField( + model_name='media', + name='manual_skip', + field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\'t be downloaded', verbose_name='manual_skip'), + ), + migrations.AddField( + model_name='media', + name='title', + field=models.CharField(blank=True, default='', help_text='Video title', max_length=200, verbose_name='title'), + ), + migrations.AddField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=200, verbose_name='filter string'), + ), + migrations.AddField( + model_name='source', + name='index_videos', + field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + migrations.AddField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), + ), + migrations.AddField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField(all_choice='all', all_label='(All Categories)', allow_all=True, default='all', help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', max_length=128, possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''), + ), + migrations.AlterField( + model_name='source', + name='source_resolution', + field=models.CharField(choices=[('audio', 'Audio only'), ('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), + ), + migrations.AlterField( + model_name='mediaserver', + name='options', + field=models.TextField(help_text='JSON encoded options for the media server', null=True, verbose_name='options'), + ), + migrations.AlterField( + model_name='mediaserver', + name='server_type', + field=models.CharField(choices=[('j', 'Jellyfin'), ('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type'), + ), + migrations.AlterField( + model_name='mediaserver', + name='use_https', + field=models.BooleanField(default=False, help_text='Connect to the media server over HTTPS', verbose_name='use https'), + ), + migrations.AlterField( + model_name='mediaserver', + name='verify_https', + field=models.BooleanField(default=True, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https'), + ), + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9'), ('AV1', 'AV1')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + migrations.RunPython( + code=fix_media_file, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/tubesync/sync/migrations/0002_auto_20201213_0817.py b/tubesync/sync/migrations/0002_auto_20201213_0817.py deleted file mode 100644 index ab2f71e3..00000000 --- a/tubesync/sync/migrations/0002_auto_20201213_0817.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-13 08:17 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - ] diff --git a/tubesync/sync/migrations/0003_source_copy_thumbnails.py b/tubesync/sync/migrations/0003_source_copy_thumbnails.py deleted file mode 100644 index 7bdbfdbc..00000000 --- a/tubesync/sync/migrations/0003_source_copy_thumbnails.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-18 01:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0002_auto_20201213_0817'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='copy_thumbnails', - field=models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails'), - ), - ] diff --git a/tubesync/sync/migrations/0004_source_media_format.py b/tubesync/sync/migrations/0004_source_media_format.py deleted file mode 100644 index f79a7036..00000000 --- a/tubesync/sync/migrations/0004_source_media_format.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-18 01:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0003_source_copy_thumbnails'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0005_auto_20201219_0312.py b/tubesync/sync/migrations/0005_auto_20201219_0312.py deleted file mode 100644 index 11ee8e27..00000000 --- a/tubesync/sync/migrations/0005_auto_20201219_0312.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 03:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0004_source_media_format'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_type', - field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'), - ), - ] diff --git a/tubesync/sync/migrations/0006_source_write_nfo.py b/tubesync/sync/migrations/0006_source_write_nfo.py deleted file mode 100644 index d5fe9365..00000000 --- a/tubesync/sync/migrations/0006_source_write_nfo.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 03:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0005_auto_20201219_0312'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_nfo', - field=models.BooleanField(default=False, help_text='Write an NFO file with the media, these may be detected and used by some media servers', verbose_name='write nfo'), - ), - ] diff --git a/tubesync/sync/migrations/0007_auto_20201219_0645.py b/tubesync/sync/migrations/0007_auto_20201219_0645.py deleted file mode 100644 index c757d679..00000000 --- a/tubesync/sync/migrations/0007_auto_20201219_0645.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 06:45 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0006_source_write_nfo'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='write_nfo', - field=models.BooleanField(default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo'), - ), - ] diff --git a/tubesync/sync/migrations/0008_source_download_cap.py b/tubesync/sync/migrations/0008_source_download_cap.py deleted file mode 100644 index 4b3ec19f..00000000 --- a/tubesync/sync/migrations/0008_source_download_cap.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.4 on 2020-12-19 06:59 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0007_auto_20201219_0645'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='download_cap', - field=models.IntegerField(choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap'), - ), - ] diff --git a/tubesync/sync/migrations/0009_auto_20210218_0442.py b/tubesync/sync/migrations/0009_auto_20210218_0442.py deleted file mode 100644 index 45b94500..00000000 --- a/tubesync/sync/migrations/0009_auto_20210218_0442.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 3.1.6 on 2021-02-18 04:42 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0008_source_download_cap'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='download_media', - field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'), - ), - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - migrations.AlterField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0010_auto_20210924_0554.py b/tubesync/sync/migrations/0010_auto_20210924_0554.py deleted file mode 100644 index 00160610..00000000 --- a/tubesync/sync/migrations/0010_auto_20210924_0554.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 3.2.7 on 2021-09-24 05:54 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0009_auto_20210218_0442'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - migrations.AlterField( - model_name='source', - name='index_schedule', - field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'), - ), - migrations.AlterField( - model_name='source', - name='media_format', - field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), - ), - ] diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py deleted file mode 100644 index 96d9f4a7..00000000 --- a/tubesync/sync/migrations/0011_auto_20220201_1654.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 3.2.11 on 2022-02-01 16:54 - -import django.core.files.storage -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0010_auto_20210924_0554'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_json', - field=models.BooleanField( - default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'), - ), - ] diff --git a/tubesync/sync/migrations/0012_alter_media_downloaded_format.py b/tubesync/sync/migrations/0012_alter_media_downloaded_format.py deleted file mode 100644 index 3e733efb..00000000 --- a/tubesync/sync/migrations/0012_alter_media_downloaded_format.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.12 on 2022-04-06 06:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0011_auto_20220201_1654'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='downloaded_format', - field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), - ), - ] diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py deleted file mode 100644 index c9eee22e..00000000 --- a/tubesync/sync/migrations/0013_fix_elative_media_file.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 3.2.12 on 2022-04-06 06:19 - -from django.conf import settings -from django.db import migrations, models - - -def fix_media_file(apps, schema_editor): - Media = apps.get_model('sync', 'Media') - for media in Media.objects.filter(downloaded=True): - download_dir = str(settings.DOWNLOAD_ROOT) - - if media.media_file.name.startswith(download_dir): - media.media_file.name = media.media_file.name[len(download_dir) + 1:] - media.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0012_alter_media_downloaded_format'), - ] - - operations = [ - migrations.RunPython(fix_media_file) - ] diff --git a/tubesync/sync/migrations/0014_alter_media_media_file.py b/tubesync/sync/migrations/0014_alter_media_media_file.py deleted file mode 100644 index 530c8e81..00000000 --- a/tubesync/sync/migrations/0014_alter_media_media_file.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 3.2.15 on 2022-12-28 20:33 - -import django.core.files.storage -from django.conf import settings -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0013_fix_elative_media_file'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - ] diff --git a/tubesync/sync/migrations/0015_auto_20230213_0603.py b/tubesync/sync/migrations/0015_auto_20230213_0603.py deleted file mode 100644 index 54592f9d..00000000 --- a/tubesync/sync/migrations/0015_auto_20230213_0603.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.2.17 on 2023-02-13 06:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0014_alter_media_media_file'), - ] - - operations = [ - migrations.AddField( - model_name='media', - name='manual_skip', - field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'), - ), - migrations.AlterField( - model_name='media', - name='skip', - field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), - ), - ] diff --git a/tubesync/sync/migrations/0016_auto_20230214_2052.py b/tubesync/sync/migrations/0016_auto_20230214_2052.py deleted file mode 100644 index ffba1952..00000000 --- a/tubesync/sync/migrations/0016_auto_20230214_2052.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 3.2.18 on 2023-02-14 20:52 - -from django.db import migrations, models -import sync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0015_auto_20230213_0603'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='embed_metadata', - field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), - ), - migrations.AddField( - model_name='source', - name='embed_thumbnail', - field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), - ), - migrations.AddField( - model_name='source', - name='enable_sponsorblock', - field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), - ), - migrations.AddField( - model_name='source', - name='sponsorblock_categories', - field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), - ), - ] diff --git a/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py deleted file mode 100644 index cc9d9578..00000000 --- a/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.18 on 2023-02-20 02:23 - -from django.db import migrations -import sync.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0016_auto_20230214_2052'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''), - ), - ] diff --git a/tubesync/sync/migrations/0018_source_subtitles.py b/tubesync/sync/migrations/0018_source_subtitles.py deleted file mode 100644 index c526b994..00000000 --- a/tubesync/sync/migrations/0018_source_subtitles.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0017_alter_source_sponsorblock_categories'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_subtitles', - field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), - ), - migrations.AddField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'), - ), - migrations.AddField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30), - ), - ] diff --git a/tubesync/sync/migrations/0019_add_delete_removed_media.py b/tubesync/sync/migrations/0019_add_delete_removed_media.py deleted file mode 100644 index 0762be87..00000000 --- a/tubesync/sync/migrations/0019_add_delete_removed_media.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0018_source_subtitles'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='delete_removed_media', - field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), - ), - ] diff --git a/tubesync/sync/migrations/0020_auto_20231024_1825.py b/tubesync/sync/migrations/0020_auto_20231024_1825.py deleted file mode 100644 index 295339a8..00000000 --- a/tubesync/sync/migrations/0020_auto_20231024_1825.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.22 on 2023-10-24 17:25 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0019_add_delete_removed_media'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), - ), - migrations.AlterField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), - ), - migrations.AlterField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), - ), - ] diff --git a/tubesync/sync/migrations/0021_source_copy_channel_images.py b/tubesync/sync/migrations/0021_source_copy_channel_images.py deleted file mode 100644 index 5d568925..00000000 --- a/tubesync/sync/migrations/0021_source_copy_channel_images.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by nothing. Done manually by InterN0te on 2023-12-10 16:36 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0020_auto_20231024_1825'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='copy_channel_images', - field=models.BooleanField(default=False, help_text='Copy channel banner and avatar. These may be detected and used by some media servers', verbose_name='copy channel images'), - ), - ] diff --git a/tubesync/sync/migrations/0022_add_delete_files_on_disk.py b/tubesync/sync/migrations/0022_add_delete_files_on_disk.py deleted file mode 100644 index 959f3d8b..00000000 --- a/tubesync/sync/migrations/0022_add_delete_files_on_disk.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0021_source_copy_channel_images'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='delete_files_on_disk', - field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), - ), - ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0023_media_duration_filter.py b/tubesync/sync/migrations/0023_media_duration_filter.py deleted file mode 100644 index 558cc0ef..00000000 --- a/tubesync/sync/migrations/0023_media_duration_filter.py +++ /dev/null @@ -1,62 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("sync", "0022_add_delete_files_on_disk"), - ] - - operations = [ - migrations.AddField( - model_name="media", - name="title", - field=models.CharField( - verbose_name="title", - max_length=100, - blank=True, - null=False, - default="", - help_text="Video title", - ), - ), - migrations.AddField( - model_name="media", - name="duration", - field=models.PositiveIntegerField( - verbose_name="duration", - blank=True, - null=True, - help_text="Duration of media in seconds", - ), - ), - migrations.AddField( - model_name="source", - name="filter_seconds", - field=models.PositiveIntegerField( - verbose_name="filter seconds", - blank=True, - null=True, - help_text="Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering", - ), - ), - migrations.AddField( - model_name="source", - name="filter_seconds_min", - field=models.BooleanField( - verbose_name="filter seconds min/max", - choices=[(True, "Minimum Length"), (False, "Maximum Length")], - default=True, - help_text="When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (" - "video greater than maximum) video duration", - ), - ), - migrations.AddField( - model_name="source", - name="filter_text_invert", - field=models.BooleanField( - verbose_name="invert filter text matching", - default=False, - help_text="Invert filter string regex match, skip any matching titles when selected", - ), - ), - ] diff --git a/tubesync/sync/migrations/0024_auto_20240717_1535.py b/tubesync/sync/migrations/0024_auto_20240717_1535.py deleted file mode 100644 index c2d20d35..00000000 --- a/tubesync/sync/migrations/0024_auto_20240717_1535.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 3.2.25 on 2024-07-17 15:35 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0023_media_duration_filter'), - ] - - operations = [ - migrations.AlterField( - model_name='media', - name='manual_skip', - field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\'t be downloaded', verbose_name='manual_skip'), - ), - migrations.AlterField( - model_name='media', - name='title', - field=models.CharField(blank=True, default='', help_text='Video title', max_length=200, verbose_name='title'), - ), - migrations.AlterField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=200, verbose_name='filter string'), - ), - ] diff --git a/tubesync/sync/migrations/0025_add_video_type_support.py b/tubesync/sync/migrations/0025_add_video_type_support.py deleted file mode 100644 index e1a901c0..00000000 --- a/tubesync/sync/migrations/0025_add_video_type_support.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0024_auto_20240717_1535'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='index_videos', - field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), - ), - migrations.AddField( - model_name='source', - name='index_streams', - field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), - ), - ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0026_alter_source_sub_langs.py b/tubesync/sync/migrations/0026_alter_source_sub_langs.py deleted file mode 100644 index 937c3a14..00000000 --- a/tubesync/sync/migrations/0026_alter_source_sub_langs.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.25 on 2024-12-11 12:43 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0025_add_video_type_support'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), - ), - ] diff --git a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py deleted file mode 100644 index c81b8e72..00000000 --- a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.2.25 on 2025-01-29 06:14 - -from django.db import migrations -import sync.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0026_alter_source_sub_langs'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(all_choice='all', all_label='(All Categories)', allow_all=True, default='all', help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', max_length=128, possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''), - ), - ] diff --git a/tubesync/sync/migrations/0028_alter_source_source_resolution.py b/tubesync/sync/migrations/0028_alter_source_source_resolution.py deleted file mode 100644 index e72f7307..00000000 --- a/tubesync/sync/migrations/0028_alter_source_source_resolution.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.2.25 on 2025-02-12 18:31 - -from django.db import migrations, models - -class Migration(migrations.Migration): - dependencies = [ - ('sync', '0027_alter_source_sponsorblock_categories'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_resolution', - field=models.CharField(choices=[('audio', 'Audio only'), ('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), - ), - ] - diff --git a/tubesync/sync/migrations/0029_alter_mediaserver_fields.py b/tubesync/sync/migrations/0029_alter_mediaserver_fields.py deleted file mode 100644 index b7363430..00000000 --- a/tubesync/sync/migrations/0029_alter_mediaserver_fields.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 3.2.25 on 2025-02-22 03:52 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0028_alter_source_source_resolution'), - ] - - operations = [ - migrations.AlterField( - model_name='mediaserver', - name='options', - field=models.TextField(help_text='JSON encoded options for the media server', null=True, verbose_name='options'), - ), - migrations.AlterField( - model_name='mediaserver', - name='server_type', - field=models.CharField(choices=[('j', 'Jellyfin'), ('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type'), - ), - migrations.AlterField( - model_name='mediaserver', - name='use_https', - field=models.BooleanField(default=False, help_text='Connect to the media server over HTTPS', verbose_name='use https'), - ), - migrations.AlterField( - model_name='mediaserver', - name='verify_https', - field=models.BooleanField(default=True, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https'), - ), - ] diff --git a/tubesync/sync/migrations/0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0030_alter_source_source_vcodec.py deleted file mode 100644 index 2b4f3618..00000000 --- a/tubesync/sync/migrations/0030_alter_source_source_vcodec.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.8 on 2025-04-07 18:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0029_alter_mediaserver_fields'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='source_vcodec', - field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9'), ('AV1', 'AV1')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), - ), - ] From 813dc075b6aba2d1ece853c7b375bd18747e8ac2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 14:44:04 -0400 Subject: [PATCH 138/454] Rewrite warning in README.md --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e7568535..0894f736 100644 --- a/README.md +++ b/README.md @@ -275,19 +275,19 @@ long as possible, up to 24 hours. ### 3. Indexing massive channels -If you add a massive (several thousand videos) channel to TubeSync and choose "index -every hour" or similar short interval it's entirely possible your TubeSync install may -spend its entire time just indexing the massive channel over and over again without +If you add a massive channel (one with several thousand videos) to TubeSync and choose "index +every hour" or a similarly short interval; it's entirely possible that your TubeSync install may +spend its entire time indexing the channel, over and over again, without downloading any media. Check your tasks for the status of your TubeSync install. -If you add a significant amount of "work" due to adding many large channels you may -need to increase the number of background workers by setting the `TUBESYNC_WORKERS` -environment variable. Try around ~4 at most, although the absolute maximum allowed is 8. - -**Be nice.** it's likely entirely possible your IP address could get throttled by the -source if you try and crawl extremely large amounts very quickly. **Try and be polite +**Be nice.** It's entirely possible that your IP address could get throttled and/or banned, by the +source, if you try to crawl extremely large amounts quickly. **Try and be polite with the smallest amount of indexing and concurrent downloads possible for your needs.** +Only, if you absolutely must, should you increase [`TUBESYNC_WORKERS`](#advanced-configuration) above its default value. +The maximum the software accepts is `8` threads per queue worker process. +By default, up to `3` tasks will be executing concurrently. +The maximum is `24` concurrent tasks. # FAQ From d28bffa6788ad7318cdaf0ea5ac0065474f546c4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Apr 2025 14:48:47 -0400 Subject: [PATCH 139/454] fixup: and => to --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0894f736..502abf3a 100644 --- a/README.md +++ b/README.md @@ -281,7 +281,7 @@ spend its entire time indexing the channel, over and over again, without downloading any media. Check your tasks for the status of your TubeSync install. **Be nice.** It's entirely possible that your IP address could get throttled and/or banned, by the -source, if you try to crawl extremely large amounts quickly. **Try and be polite +source, if you try to crawl extremely large amounts quickly. **Try to be polite with the smallest amount of indexing and concurrent downloads possible for your needs.** Only, if you absolutely must, should you increase [`TUBESYNC_WORKERS`](#advanced-configuration) above its default value. From ba130278d3563295845dc82a1514743ae06123b5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 14:09:57 -0400 Subject: [PATCH 140/454] Update dependency in 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 381dace4..00efa0f6 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -9,7 +9,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0030_alter_source_source_vcodec'), + ('sync', '0001_squashed_0030_alter_source_source_vcodec'), ] operations = [ From 347c13d54d17d0297bac3d618b69931c5a7d6816 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 15:14:39 -0400 Subject: [PATCH 141/454] Update table names for metadata --- tubesync/sync/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index faccd565..92978c4d 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1745,11 +1745,13 @@ class Metadata(models.Model): Metadata for an indexed `Media` item. ''' class Meta: + db_table = 'sync_media_metadata' verbose_name = _('Metadata about a Media item') verbose_name_plural = _('Metadata about a Media item') unique_together = ( ('media', 'site', 'key'), ) + get_latest_by = ["-retrieved", "-created"] uuid = models.UUIDField( _('uuid'), @@ -1872,12 +1874,14 @@ class MetadataFormat(models.Model): A format from the Metadata for an indexed `Media` item. ''' class Meta: + db_table = f'{Metadata._meta.db_table}_format' verbose_name = _('Format from the Metadata about a Media item') verbose_name_plural = _('Formats from the Metadata about a Media item') unique_together = ( ('metadata', 'site', 'key', 'number'), ('metadata', 'site', 'key', 'code'), ) + ordering = ['site', 'key', 'number'] uuid = models.UUIDField( _('uuid'), From f386416948d9fc8757a51d3e476b65d8647c9260 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 15:26:51 -0400 Subject: [PATCH 142/454] Update and rename 0031_metadata_metadataformat.py to 0031_squashed_metadata_metadataformat.py --- ... => 0031_squashed_metadata_metadataformat.py} | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) rename tubesync/sync/migrations/{0031_metadata_metadataformat.py => 0031_squashed_metadata_metadataformat.py} (87%) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py similarity index 87% rename from tubesync/sync/migrations/0031_metadata_metadataformat.py rename to tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index 00bcec08..d66189e8 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.8 on 2025-04-11 07:36 +# Generated by Django 5.1.8 on 2025-04-20 19:10 import django.db.models.deletion import sync.models @@ -8,8 +8,10 @@ from django.db import migrations, models class Migration(migrations.Migration): + replaces = [('sync', '0031_metadata_metadataformat')] + dependencies = [ - ('sync', '0030_alter_source_source_vcodec'), + ('sync', '0001_squashed_0030_alter_source_source_vcodec'), ] operations = [ @@ -30,6 +32,7 @@ class Migration(migrations.Migration): 'verbose_name': 'Metadata about a Media item', 'verbose_name_plural': 'Metadata about a Media item', 'unique_together': {('media', 'site', 'key')}, + 'get_latest_by': ['-retrieved', '-created'], }, ), migrations.CreateModel( @@ -47,6 +50,15 @@ class Migration(migrations.Migration): 'verbose_name': 'Format from the Metadata about a Media item', 'verbose_name_plural': 'Formats from the Metadata about a Media item', 'unique_together': {('metadata', 'site', 'key', 'code'), ('metadata', 'site', 'key', 'number')}, + 'ordering': ['site', 'key', 'number'], }, ), + migrations.AlterModelTable( + name='metadata', + table='sync_media_metadata', + ), + migrations.AlterModelTable( + name='metadataformat', + table='sync_media_metadata_format', + ), ] From 4714336f28dba0af85192ab6e98a2194b516b147 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 15:30:50 -0400 Subject: [PATCH 143/454] Add files via upload --- .../0031_metadata_metadataformat.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tubesync/sync/migrations/0031_metadata_metadataformat.py diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py new file mode 100644 index 00000000..381dace4 --- /dev/null +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -0,0 +1,52 @@ +# Generated by Django 5.1.8 on 2025-04-11 07:36 + +import django.db.models.deletion +import sync.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0030_alter_source_source_vcodec'), + ] + + operations = [ + migrations.CreateModel( + name='Metadata', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the metadata', primary_key=True, serialize=False, verbose_name='uuid')), + ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was created', verbose_name='created')), + ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), + ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), + ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), + ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), + ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), + ], + options={ + 'verbose_name': 'Metadata about a Media item', + 'verbose_name_plural': 'Metadata about a Media item', + 'unique_together': {('media', 'site', 'key')}, + }, + ), + migrations.CreateModel( + name='MetadataFormat', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the format', primary_key=True, serialize=False, verbose_name='uuid')), + ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), + ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), + ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), + ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), + ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), + ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), + ], + options={ + 'verbose_name': 'Format from the Metadata about a Media item', + 'verbose_name_plural': 'Formats from the Metadata about a Media item', + 'unique_together': {('metadata', 'site', 'key', 'code'), ('metadata', 'site', 'key', 'number')}, + }, + ), + ] From eb396aaf3cadfe66f52d1deb3d6d5c19bf18e0f0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 20:05:22 -0400 Subject: [PATCH 144/454] Prefer `release_timestamp` when it is available --- tubesync/sync/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 92978c4d..e8ae8867 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1275,7 +1275,9 @@ class Media(models.Model): def metadata_published(self, timestamp=None): if timestamp is None: - timestamp = self.get_metadata_first_value('timestamp') + timestamp = self.get_metadata_first_value( + ('release_timestamp', 'timestamp',) + ) return self.ts_to_dt(timestamp) @property From fe60012cfc04a96fc92be3a357b96efaeee15111 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 20 Apr 2025 22:12:53 -0400 Subject: [PATCH 145/454] Add `metadata_dumps` function --- tubesync/sync/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e8ae8867..8a065f4e 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1104,6 +1104,11 @@ class Media(models.Model): return self.metadata is not None + def metadata_dumps(self, arg_dict=dict()): + data = arg_dict or self.new_metadata.with_formats + return json.dumps(data, separators=(',', ':'), default=json_serial) + + def metadata_loads(self, arg_str='{}'): data = json.loads(arg_str) or self.loaded_metadata return self.ingest_metadata(data) From 86e9ac2e342032f92efc82e9092b2248bbe24857 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 12:30:10 -0400 Subject: [PATCH 146/454] Start `0.15` with the new metadata --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 37ca2699..dac5896f 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.14.1' +VERSION = '0.15.0' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From 2a10ecce76fc0faaee7c99b50436afb0495f3c90 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 14:09:41 -0400 Subject: [PATCH 147/454] Update models.py --- tubesync/sync/models.py | 57 +++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 8a065f4e..85c7af1e 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -19,7 +19,8 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException -from common.utils import clean_filename, clean_emoji +from common.utils import ( clean_filename, clean_emoji, + django_queryset_generator as qs_gen, ) from .youtube import (get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) @@ -812,6 +813,7 @@ class Media(models.Model): ) def save(self, force_insert=False, force_update=False, using=None, update_fields=None): + setattr(self, '_cached_metadata_dict', None) # Correct the path after a source is renamed if self.created and self.downloaded and not self.media_file_exists: fp_list = list((self.filepath,)) @@ -838,9 +840,9 @@ class Media(models.Model): ) ) if update_md: + self.title = self.metadata_title[:200] or self.title + self.duration = self.metadata_duration or self.duration setattr(self, '_cached_metadata_dict', None) - self.title = self.metadata_title[:200] - self.duration = self.metadata_duration if update_fields is not None: # If only some fields are being updated, make sure we update title and duration if metadata changes update_fields = {"title", "duration"}.union(update_fields) @@ -1105,6 +1107,7 @@ class Media(models.Model): def metadata_dumps(self, arg_dict=dict()): + from common.utils import json_serial data = arg_dict or self.new_metadata.with_formats return json.dumps(data, separators=(',', ':'), default=json_serial) @@ -1135,13 +1138,11 @@ class Media(models.Model): def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value - self.ingest_metadata(data) #epoch = self.get_metadata_first_value('epoch', arg_dict=data) #migrated = dict(migrated=True, epoch=epoch) - from common.utils import json_serial - compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) - self.metadata = compact_json + self.metadata = self.metadata_dumps(arg_dict=data) self.save() + self.ingest_metadata(data) from common.logger import log log.debug(f'Saved to metadata: {self.key} / {self.uuid}: {key=}: {value}') @@ -1158,12 +1159,11 @@ class Media(models.Model): if (now - ran_at) < timedelta(hours=1): return data - from common.utils import json_serial - compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) + compact_json = self.metadata_dumps(arg_dict=data) filtered_data = filter_response(data, True) filtered_data['_reduce_data_ran_at'] = round((now - self.posix_epoch).total_seconds()) - filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial) + filtered_json = self.metadata_dumps(arg_dict=filtered_data) except Exception as e: from common.logger import log log.exception('reduce_data: %s', e) @@ -1753,8 +1753,8 @@ class Metadata(models.Model): ''' class Meta: db_table = 'sync_media_metadata' - verbose_name = _('Metadata about a Media item') - verbose_name_plural = _('Metadata about a Media item') + verbose_name = _('Metadata about Media') + verbose_name_plural = _('Metadata about Media') unique_together = ( ('media', 'site', 'key'), ) @@ -1770,10 +1770,10 @@ class Metadata(models.Model): media = models.OneToOneField( Media, # on_delete=models.DO_NOTHING, - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name='new_metadata', help_text=_('Media the metadata belongs to'), - null=False, + null=True, parent_link=False, ) site = models.CharField( @@ -1830,14 +1830,14 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_formats(self, formats=list(), /): for number, format in enumerate(formats, start=1): - mdf = self.metadataformat.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0] + mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) mdf.value = format mdf.save() @property def with_formats(self): - formats = self.metadataformat.all().order_by('number') - formats_list = [ f.value for f in formats ] + formats = self.format.all().order_by('number') + formats_list = [ f.value for f in qs_gen(formats) ] metadata = self.value.copy() metadata.update(dict(formats=formats_list)) return metadata @@ -1869,7 +1869,11 @@ class Metadata(models.Model): self.value = data.copy() # try not to have side-effects for the caller formats_key = self.media.get_metadata_field('formats') formats = self.value.pop(formats_key, list()) - self.uploaded = self.media.published + self.uploaded = min( + self.published, + self.retrieved, + self.media.created, + ) self.save() self.ingest_formats(formats) @@ -1882,11 +1886,10 @@ class MetadataFormat(models.Model): ''' class Meta: db_table = f'{Metadata._meta.db_table}_format' - verbose_name = _('Format from the Metadata about a Media item') - verbose_name_plural = _('Formats from the Metadata about a Media item') + verbose_name = _('Format from Media Metadata') + verbose_name_plural = _('Formats from Media Metadata') unique_together = ( ('metadata', 'site', 'key', 'number'), - ('metadata', 'site', 'key', 'code'), ) ordering = ['site', 'key', 'number'] @@ -1901,7 +1904,7 @@ class MetadataFormat(models.Model): Metadata, # on_delete=models.DO_NOTHING, on_delete=models.CASCADE, - related_name='metadataformat', + related_name='format', help_text=_('Metadata the format belongs to'), null=False, ) @@ -1921,7 +1924,7 @@ class MetadataFormat(models.Model): db_index=True, null=False, default='', - help_text=_('Media identifier at the site for which this format is available'), + help_text=_('Media identifier at the site from which this format is available'), ) number = models.PositiveIntegerField( _('number'), @@ -1929,14 +1932,6 @@ class MetadataFormat(models.Model): null=False, help_text=_('Ordering number for this format') ) - code = models.CharField( - _('code'), - max_length=64, - blank=True, - null=False, - default='', - help_text=_('Format identification code'), - ) value = models.JSONField( _('value'), encoder=JSONEncoder, From f87ffdf104ec0b20e31697e09a365c57d68caf65 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 14:11:11 -0400 Subject: [PATCH 148/454] Remove `code` --- tubesync/sync/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/admin.py b/tubesync/sync/admin.py index f4ecfbc0..b1e7dbaf 100644 --- a/tubesync/sync/admin.py +++ b/tubesync/sync/admin.py @@ -40,7 +40,7 @@ class MetadataAdmin(admin.ModelAdmin): class MetadataFormatAdmin(admin.ModelAdmin): ordering = ('site', 'key', 'number') - list_display = ('uuid', 'key', 'site', 'code', 'number', 'metadata') + list_display = ('uuid', 'key', 'site', 'number', 'metadata') readonly_fields = ('uuid', 'metadata', 'site', 'key', 'number') search_fields = ('uuid', 'metadata__uuid', 'metadata__media__uuid', 'key') From 7ae75edffc371dc4643282b05c0611aa9e1d3c9c Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 15:12:36 -0400 Subject: [PATCH 149/454] Create timestamp.py --- tubesync/common/timestamp.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tubesync/common/timestamp.py diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py new file mode 100644 index 00000000..833c488f --- /dev/null +++ b/tubesync/common/timestamp.py @@ -0,0 +1,35 @@ +import datetime +from django.utils import timezone + + +posix_epoch = datetime.datetime.utcfromtimestamp(0) +utc_tz = datetime.timezone.utc + + +def add_epoch(seconds): + assert seconds is not None + assert seconds >= 0, 'seconds must be a positive number' + + return timedelta(seconds=seconds) + posix_epoch + +def subtract_epoch(arg_dt, /): + epoch = posix_epoch.astimezone(utc_tz) + utc_dt = arg_dt.astimezone(utc_tz) + + return utc_dt - epoch + +def datetime_to_timestamp(arg_dt, /): + timestamp = subtract_epoch(arg_dt).total_seconds() + + try: + timestamp_int = int(timestamp) + except (TypeError, ValueError,): + pass + else: + return timestamp_int + + return timestamp + +def timestamp_to_datetime(seconds, /): + return add_epoch(seconds=seconds).astimezone(utc_tz) + From f8a2eeeb97ddefb3d5c192a43200ba680f749d17 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 15:25:58 -0400 Subject: [PATCH 150/454] Simplify `datetime_to_timestamp` --- tubesync/common/timestamp.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index 833c488f..b36e32e2 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -18,17 +18,12 @@ def subtract_epoch(arg_dt, /): return utc_dt - epoch -def datetime_to_timestamp(arg_dt, /): +def datetime_to_timestamp(arg_dt, /, *, integer=True): timestamp = subtract_epoch(arg_dt).total_seconds() - try: - timestamp_int = int(timestamp) - except (TypeError, ValueError,): - pass - else: - return timestamp_int - - return timestamp + if not integer: + return timestamp + return int(timestamp) def timestamp_to_datetime(seconds, /): return add_epoch(seconds=seconds).astimezone(utc_tz) From 55cc637d9febc16838d92c11b53119b26e101965 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 15:40:17 -0400 Subject: [PATCH 151/454] Use `timestamp_to_datetime` function --- tubesync/sync/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 85c7af1e..5474dcc6 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1845,9 +1845,10 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_metadata(self, data): assert isinstance(data, dict), type(data) + from common.timestamp import timestamp_to_datetime try: - self.retrieved = self.media.ts_to_dt( + self.retrieved = timestamp_to_datetime( self.media.get_metadata_first_value( 'epoch', arg_dict=data, @@ -1857,7 +1858,7 @@ class Metadata(models.Model): self.retrieved = self.created try: - self.published = self.media.ts_to_dt( + self.published = timestamp_to_datetime( self.media.get_metadata_first_value( ('release_timestamp', 'timestamp',), arg_dict=data, From 023945d10e88a900defd246fc56c8c61b7438b11 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 17:00:18 -0400 Subject: [PATCH 152/454] Restore old migration files (25-30) --- .../migrations/0025_add_video_type_support.py | 20 +++++++++++ .../migrations/0026_alter_source_sub_langs.py | 19 +++++++++++ ...27_alter_source_sponsorblock_categories.py | 19 +++++++++++ .../0028_alter_source_source_resolution.py | 17 ++++++++++ .../0029_alter_mediaserver_fields.py | 33 +++++++++++++++++++ .../0030_alter_source_source_vcodec.py | 18 ++++++++++ 6 files changed, 126 insertions(+) create mode 100644 tubesync/sync/migrations/0025_add_video_type_support.py create mode 100644 tubesync/sync/migrations/0026_alter_source_sub_langs.py create mode 100644 tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py create mode 100644 tubesync/sync/migrations/0028_alter_source_source_resolution.py create mode 100644 tubesync/sync/migrations/0029_alter_mediaserver_fields.py create mode 100644 tubesync/sync/migrations/0030_alter_source_source_vcodec.py diff --git a/tubesync/sync/migrations/0025_add_video_type_support.py b/tubesync/sync/migrations/0025_add_video_type_support.py new file mode 100644 index 00000000..e1a901c0 --- /dev/null +++ b/tubesync/sync/migrations/0025_add_video_type_support.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0024_auto_20240717_1535'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='index_videos', + field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0026_alter_source_sub_langs.py b/tubesync/sync/migrations/0026_alter_source_sub_langs.py new file mode 100644 index 00000000..937c3a14 --- /dev/null +++ b/tubesync/sync/migrations/0026_alter_source_sub_langs.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-12-11 12:43 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0025_add_video_type_support'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), + ), + ] diff --git a/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py new file mode 100644 index 00000000..c81b8e72 --- /dev/null +++ b/tubesync/sync/migrations/0027_alter_source_sponsorblock_categories.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2025-01-29 06:14 + +from django.db import migrations +import sync.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0026_alter_source_sub_langs'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField(all_choice='all', all_label='(All Categories)', allow_all=True, default='all', help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', max_length=128, possible_choices=[('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section')], verbose_name=''), + ), + ] diff --git a/tubesync/sync/migrations/0028_alter_source_source_resolution.py b/tubesync/sync/migrations/0028_alter_source_source_resolution.py new file mode 100644 index 00000000..e72f7307 --- /dev/null +++ b/tubesync/sync/migrations/0028_alter_source_source_resolution.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.25 on 2025-02-12 18:31 + +from django.db import migrations, models + +class Migration(migrations.Migration): + dependencies = [ + ('sync', '0027_alter_source_sponsorblock_categories'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_resolution', + field=models.CharField(choices=[('audio', 'Audio only'), ('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), + ), + ] + diff --git a/tubesync/sync/migrations/0029_alter_mediaserver_fields.py b/tubesync/sync/migrations/0029_alter_mediaserver_fields.py new file mode 100644 index 00000000..b7363430 --- /dev/null +++ b/tubesync/sync/migrations/0029_alter_mediaserver_fields.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.25 on 2025-02-22 03:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0028_alter_source_source_resolution'), + ] + + operations = [ + migrations.AlterField( + model_name='mediaserver', + name='options', + field=models.TextField(help_text='JSON encoded options for the media server', null=True, verbose_name='options'), + ), + migrations.AlterField( + model_name='mediaserver', + name='server_type', + field=models.CharField(choices=[('j', 'Jellyfin'), ('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type'), + ), + migrations.AlterField( + model_name='mediaserver', + name='use_https', + field=models.BooleanField(default=False, help_text='Connect to the media server over HTTPS', verbose_name='use https'), + ), + migrations.AlterField( + model_name='mediaserver', + name='verify_https', + field=models.BooleanField(default=True, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https'), + ), + ] diff --git a/tubesync/sync/migrations/0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0030_alter_source_source_vcodec.py new file mode 100644 index 00000000..2b4f3618 --- /dev/null +++ b/tubesync/sync/migrations/0030_alter_source_source_vcodec.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.8 on 2025-04-07 18:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0029_alter_mediaserver_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9'), ('AV1', 'AV1')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + ] From 7b3ee3ba95309a28ec369d44cd5cbbdf0ebaaada Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 17:01:32 -0400 Subject: [PATCH 153/454] Restore old migration files (20-24) --- .../migrations/0020_auto_20231024_1825.py | 29 +++++++++ .../0021_source_copy_channel_images.py | 18 ++++++ .../0022_add_delete_files_on_disk.py | 17 +++++ .../migrations/0023_media_duration_filter.py | 62 +++++++++++++++++++ .../migrations/0024_auto_20240717_1535.py | 28 +++++++++ 5 files changed, 154 insertions(+) create mode 100644 tubesync/sync/migrations/0020_auto_20231024_1825.py create mode 100644 tubesync/sync/migrations/0021_source_copy_channel_images.py create mode 100644 tubesync/sync/migrations/0022_add_delete_files_on_disk.py create mode 100644 tubesync/sync/migrations/0023_media_duration_filter.py create mode 100644 tubesync/sync/migrations/0024_auto_20240717_1535.py diff --git a/tubesync/sync/migrations/0020_auto_20231024_1825.py b/tubesync/sync/migrations/0020_auto_20231024_1825.py new file mode 100644 index 00000000..295339a8 --- /dev/null +++ b/tubesync/sync/migrations/0020_auto_20231024_1825.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.22 on 2023-10-24 17:25 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0019_add_delete_removed_media'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), + ), + migrations.AlterField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), + ), + migrations.AlterField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), + ), + ] diff --git a/tubesync/sync/migrations/0021_source_copy_channel_images.py b/tubesync/sync/migrations/0021_source_copy_channel_images.py new file mode 100644 index 00000000..5d568925 --- /dev/null +++ b/tubesync/sync/migrations/0021_source_copy_channel_images.py @@ -0,0 +1,18 @@ +# Generated by nothing. Done manually by InterN0te on 2023-12-10 16:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0020_auto_20231024_1825'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='copy_channel_images', + field=models.BooleanField(default=False, help_text='Copy channel banner and avatar. These may be detected and used by some media servers', verbose_name='copy channel images'), + ), + ] diff --git a/tubesync/sync/migrations/0022_add_delete_files_on_disk.py b/tubesync/sync/migrations/0022_add_delete_files_on_disk.py new file mode 100644 index 00000000..959f3d8b --- /dev/null +++ b/tubesync/sync/migrations/0022_add_delete_files_on_disk.py @@ -0,0 +1,17 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0021_source_copy_channel_images'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/migrations/0023_media_duration_filter.py b/tubesync/sync/migrations/0023_media_duration_filter.py new file mode 100644 index 00000000..558cc0ef --- /dev/null +++ b/tubesync/sync/migrations/0023_media_duration_filter.py @@ -0,0 +1,62 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("sync", "0022_add_delete_files_on_disk"), + ] + + operations = [ + migrations.AddField( + model_name="media", + name="title", + field=models.CharField( + verbose_name="title", + max_length=100, + blank=True, + null=False, + default="", + help_text="Video title", + ), + ), + migrations.AddField( + model_name="media", + name="duration", + field=models.PositiveIntegerField( + verbose_name="duration", + blank=True, + null=True, + help_text="Duration of media in seconds", + ), + ), + migrations.AddField( + model_name="source", + name="filter_seconds", + field=models.PositiveIntegerField( + verbose_name="filter seconds", + blank=True, + null=True, + help_text="Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering", + ), + ), + migrations.AddField( + model_name="source", + name="filter_seconds_min", + field=models.BooleanField( + verbose_name="filter seconds min/max", + choices=[(True, "Minimum Length"), (False, "Maximum Length")], + default=True, + help_text="When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (" + "video greater than maximum) video duration", + ), + ), + migrations.AddField( + model_name="source", + name="filter_text_invert", + field=models.BooleanField( + verbose_name="invert filter text matching", + default=False, + help_text="Invert filter string regex match, skip any matching titles when selected", + ), + ), + ] diff --git a/tubesync/sync/migrations/0024_auto_20240717_1535.py b/tubesync/sync/migrations/0024_auto_20240717_1535.py new file mode 100644 index 00000000..c2d20d35 --- /dev/null +++ b/tubesync/sync/migrations/0024_auto_20240717_1535.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.25 on 2024-07-17 15:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0023_media_duration_filter'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='manual_skip', + field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\'t be downloaded', verbose_name='manual_skip'), + ), + migrations.AlterField( + model_name='media', + name='title', + field=models.CharField(blank=True, default='', help_text='Video title', max_length=200, verbose_name='title'), + ), + migrations.AlterField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=200, verbose_name='filter string'), + ), + ] From 03b85cebfcbbbc58ce1bdbb91e1989425d7184b2 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 17:33:12 -0400 Subject: [PATCH 154/454] Indentation for models.py --- tubesync/sync/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 8fd8eb5f..7156e274 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -20,15 +20,15 @@ from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException from common.utils import clean_filename, clean_emoji -from .youtube import (get_media_info as get_youtube_media_info, - download_media as download_youtube_media, - get_channel_image_info as get_youtube_channel_image_info) +from .youtube import ( get_media_info as get_youtube_media_info, + download_media as download_youtube_media, + get_channel_image_info as get_youtube_channel_image_info) from .utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote) -from .matching import (get_best_combined_format, get_best_audio_format, - get_best_video_format) +from .matching import ( get_best_combined_format, get_best_audio_format, + get_best_video_format) from .fields import CommaSepChoiceField -from .choices import (Val, CapChoices, Fallback, FileExtension, +from .choices import ( Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, From 2cad19f5ad4296602bf61b7f257d5d87907cb2fe Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 18:58:43 -0400 Subject: [PATCH 155/454] Use round for seconds --- tubesync/common/timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index b36e32e2..27398be1 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -23,7 +23,7 @@ def datetime_to_timestamp(arg_dt, /, *, integer=True): if not integer: return timestamp - return int(timestamp) + return round(timestamp) def timestamp_to_datetime(seconds, /): return add_epoch(seconds=seconds).astimezone(utc_tz) From a05968e56171502c34f12e1d9d16a30a0b10a732 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 22:36:34 -0400 Subject: [PATCH 156/454] Update timestamp.py --- tubesync/common/timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index 27398be1..e4bd6d63 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -1,5 +1,4 @@ import datetime -from django.utils import timezone posix_epoch = datetime.datetime.utcfromtimestamp(0) @@ -13,6 +12,7 @@ def add_epoch(seconds): return timedelta(seconds=seconds) + posix_epoch def subtract_epoch(arg_dt, /): + assert isinstance(arg_dt, datetime.datetime) epoch = posix_epoch.astimezone(utc_tz) utc_dt = arg_dt.astimezone(utc_tz) From 413043b7e9717483466e324935f5f2fb64760ec8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 23:25:27 -0400 Subject: [PATCH 157/454] Update 0031_squashed_metadata_metadataformat.py --- .../0031_squashed_metadata_metadataformat.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index d66189e8..622a8799 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -26,11 +26,11 @@ class Migration(migrations.Migration): ('uploaded', models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), - ('media', models.OneToOneField(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='new_metadata', to='sync.media')), + ('media', models.OneToOneField(help_text='Media the metadata belongs to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='new_metadata', to='sync.media')), ], options={ - 'verbose_name': 'Metadata about a Media item', - 'verbose_name_plural': 'Metadata about a Media item', + 'verbose_name': 'Metadata about Media', + 'verbose_name_plural': 'Metadata about Media', 'unique_together': {('media', 'site', 'key')}, 'get_latest_by': ['-retrieved', '-created'], }, @@ -40,16 +40,15 @@ class Migration(migrations.Migration): fields=[ ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the format', primary_key=True, serialize=False, verbose_name='uuid')), ('site', models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), - ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), + ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), - ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), - ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat', to='sync.metadata')), + ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata')), ], options={ - 'verbose_name': 'Format from the Metadata about a Media item', - 'verbose_name_plural': 'Formats from the Metadata about a Media item', - 'unique_together': {('metadata', 'site', 'key', 'code'), ('metadata', 'site', 'key', 'number')}, + 'verbose_name': 'Format from Media Metadata', + 'verbose_name_plural': 'Formats from Media Metadata', + 'unique_together': {('metadata', 'site', 'key', 'number')}, 'ordering': ['site', 'key', 'number'], }, ), From 6c135dbb5e27900b460b564f6430f25b3c4d9247 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 23:42:19 -0400 Subject: [PATCH 158/454] Upload squashed changes --- .../migrations/0031_squashed_metadata_metadataformat.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index 622a8799..da8f8d7a 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.8 on 2025-04-20 19:10 +# Generated by Django 5.1.8 on 2025-04-22 03:34 import django.db.models.deletion import sync.models @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): - replaces = [('sync', '0031_metadata_metadataformat')] + replaces = [('sync', '0031_metadata_metadataformat'), ('sync', '0032_alter_metadata_options_alter_metadataformat_options_and_more')] dependencies = [ ('sync', '0001_squashed_0030_alter_source_source_vcodec'), @@ -42,6 +42,7 @@ class Migration(migrations.Migration): ('site', models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')), ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), + ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata')), ], @@ -60,4 +61,8 @@ class Migration(migrations.Migration): name='metadataformat', table='sync_media_metadata_format', ), + migrations.RemoveField( + model_name='metadataformat', + name='code', + ), ] From af4693f7d4337c0cf89168597a6394ffc35674a3 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 21 Apr 2025 23:45:56 -0400 Subject: [PATCH 159/454] Update 0031_squashed_metadata_metadataformat.py --- .../sync/migrations/0031_squashed_metadata_metadataformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index da8f8d7a..71e681ff 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): - replaces = [('sync', '0031_metadata_metadataformat'), ('sync', '0032_alter_metadata_options_alter_metadataformat_options_and_more')] + replaces = [('sync', '0031_metadata_metadataformat')] dependencies = [ ('sync', '0001_squashed_0030_alter_source_source_vcodec'), From 8483e3ebb8d9f02262c1bd63cf15dcee2ffe1814 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 01:58:37 -0400 Subject: [PATCH 160/454] Update timestamp.py --- tubesync/common/timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index e4bd6d63..df7b2f13 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -9,7 +9,7 @@ def add_epoch(seconds): assert seconds is not None assert seconds >= 0, 'seconds must be a positive number' - return timedelta(seconds=seconds) + posix_epoch + return datetime.timedelta(seconds=seconds) + posix_epoch def subtract_epoch(arg_dt, /): assert isinstance(arg_dt, datetime.datetime) From 08649d40d119b1216b7627b514802a3a68a14589 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 14:11:45 -0400 Subject: [PATCH 161/454] Add a post processor for after move --- tubesync/sync/youtube.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 61f9a489..8b753ff3 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -381,10 +381,30 @@ def download_media( 'modifychapters+ffmpeg': codec_options, }) + # Provide the user control of 'overwrites' in the post processors. + pp_opts.overwrites = opts.get( + 'overwrites', + ytopts.get( + 'overwrites', + default_opts.overwrites, + ), + ) + # Create the post processors list. # It already included user configured post processors as well. ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts)) + if pp_opts.extractaudio: + ytopts['postprocessors'].append(dict( + key='Exec', + when='after_move', + exec_cmd=( + f'test -f {output_file} || ' + 'mv -f {} ' + f'{output_file}' + ), + )) + opts.update(ytopts) with yt_dlp.YoutubeDL(opts) as y: From fcc14bcd368e6fe72e0f4917d4bf86d04ca7cf90 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 15:55:22 -0400 Subject: [PATCH 162/454] Use `shell_quote` function and add `final_ext` --- tubesync/sync/youtube.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 8b753ff3..1cf658dc 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -20,7 +20,7 @@ from .utils import mkdir_p import yt_dlp import yt_dlp.patch.check_thumbnails import yt_dlp.patch.fatal_http_errors -from yt_dlp.utils import remove_end, OUTTMPL_TYPES +from yt_dlp.utils import remove_end, shell_quote, OUTTMPL_TYPES _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {}) @@ -317,6 +317,7 @@ def download_media( ytopts = { 'format': media_format, + 'final_ext': extension, 'merge_output_format': extension, 'outtmpl': os.path.basename(output_file), 'quiet': False if settings.DEBUG else True, @@ -394,14 +395,21 @@ def download_media( # It already included user configured post processors as well. ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts)) + final_path = Path(output_file) + try: + final_path = final_path.resolve(strict=True) + except FileNotFoundError: + # This is very likely the common case + final_path = Path(output_file).resolve(strict=False) if pp_opts.extractaudio: + expected_file = shell_quote(str(final_path)) ytopts['postprocessors'].append(dict( key='Exec', when='after_move', exec_cmd=( - f'test -f {output_file} || ' - 'mv -f {} ' - f'{output_file}' + f'test -f {expected_file} || ' + 'mv -T -u -- %(filepath,_filename|)q ' + f'{expected_file}' ), )) From f2201d327ecc7f2ff3e03383edd2f69f80baaf6f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 15:58:13 -0400 Subject: [PATCH 163/454] Add comments about the added post processor --- tubesync/sync/youtube.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 1cf658dc..d3c8f4a0 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -395,6 +395,9 @@ def download_media( # It already included user configured post processors as well. ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts)) + # The ExtractAudio post processor can change the extension. + # This post processor is to change the final filename back + # to what we are expecting it to be. final_path = Path(output_file) try: final_path = final_path.resolve(strict=True) From 112b477f116b58cddf71ed775f7f06dde19c80f7 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 16:39:44 -0400 Subject: [PATCH 164/454] Play nicely with `yt_dlp` options --- tubesync/sync/youtube.py | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index d3c8f4a0..ffcbb074 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -314,6 +314,27 @@ def download_media( if extension in audio_exts: pp_opts.extractaudio = True pp_opts.nopostoverwrites = False + # The ExtractAudio post processor can change the extension. + # This post processor is to change the final filename back + # to what we are expecting it to be. + final_path = Path(output_file) + try: + final_path = final_path.resolve(strict=True) + except FileNotFoundError: + # This is very likely the common case + final_path = Path(output_file).resolve(strict=False) + expected_file = shell_quote(str(final_path)) + cmds = pp_opts.exec_cmd.get('after_move', list()) + # It is important that we use a tuple for strings. + # Otherwise, list adds each character instead. + # That last comma is really necessary! + cmds += ( + f'test -f {expected_file} || ' + 'mv -T -u -- %(filepath,_filename|)q ' + f'{expected_file}', + ) + # assignment is the quickest way to cover both 'get' cases + pp_opts.exec_cmd['after_move'] = cmds ytopts = { 'format': media_format, @@ -395,27 +416,6 @@ def download_media( # It already included user configured post processors as well. ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts)) - # The ExtractAudio post processor can change the extension. - # This post processor is to change the final filename back - # to what we are expecting it to be. - final_path = Path(output_file) - try: - final_path = final_path.resolve(strict=True) - except FileNotFoundError: - # This is very likely the common case - final_path = Path(output_file).resolve(strict=False) - if pp_opts.extractaudio: - expected_file = shell_quote(str(final_path)) - ytopts['postprocessors'].append(dict( - key='Exec', - when='after_move', - exec_cmd=( - f'test -f {expected_file} || ' - 'mv -T -u -- %(filepath,_filename|)q ' - f'{expected_file}' - ), - )) - opts.update(ytopts) with yt_dlp.YoutubeDL(opts) as y: From 479471ef635f82a2ff1d5a1b128c823d22fee795 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 19:49:34 -0400 Subject: [PATCH 165/454] Use a more human-friendly `Metadata` display --- tubesync/sync/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index d6e16460..ab69d177 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1827,6 +1827,14 @@ class Metadata(models.Model): ) + def __str__(self): + template = '"{}" from {} at: {}' + return template.format( + self.key, + self.site, + self.retrieved.isoformat(timespec='seconds'), + ) + @atomic(durable=False) def ingest_formats(self, formats=list(), /): for number, format in enumerate(formats, start=1): From ead75913a8023add9e5eca53732eb2ecef7b789e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 20:06:47 -0400 Subject: [PATCH 166/454] Use a more human-friendly `MetadataFormat` display --- tubesync/sync/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index ab69d177..121690bb 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1950,6 +1950,16 @@ class MetadataFormat(models.Model): ) + def __str__(self): + template = '#{} "{}" from {}: {}' + return template.format( + self.number, + self.key, + self.site, + self.value.get('format') or self.value.get('format_id'), + ) + + class MediaServer(models.Model): ''' A remote media server, such as a Plex server. From 8bba17c2dcf3f16cae81cdcc306c93f61795da90 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 22 Apr 2025 20:35:09 -0400 Subject: [PATCH 167/454] Use number instead of decimal integer presentation --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 121690bb..000a59d3 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1951,7 +1951,7 @@ class MetadataFormat(models.Model): def __str__(self): - template = '#{} "{}" from {}: {}' + template = '#{:n} "{}" from {}: {}' return template.format( self.number, self.key, From 72b404b0fb0584c619ba0db6ae0469e4a0e1fa88 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 04:02:39 -0400 Subject: [PATCH 168/454] Adjust `download_media_metadata` task --- tubesync/sync/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2b055ce5..19424e8d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -321,6 +321,7 @@ def index_source_task(source_id): # Video has no unique key (ID), it can't be indexed continue update_task_status(task, tvn_format.format(vn)) + # media, new_media = Media.objects.get_or_create(key=key, source=source) try: media = Media.objects.get(key=key, source=source) except Media.DoesNotExist: @@ -340,6 +341,7 @@ def index_source_task(source_id): log.debug(f'Indexed media: {vn}: {source} / {media}') # log the new media instances new_media_instance = ( + # new_media or media.created and source.last_crawl and media.created >= source.last_crawl @@ -491,14 +493,15 @@ def download_media_metadata(media_id): response = metadata if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) - media.metadata = json.dumps(response, separators=(',', ':'), default=json_serial) + media.ingest_metadata(response) + media.metadata = media.metadata_dumps(arg_dict=response) upload_date = media.upload_date # Media must have a valid upload date if upload_date: media.published = timezone.make_aware(upload_date) published = media.metadata_published() if published: - media.published = published + media.published = timezone.make_aware(published) # Store title in DB so it's fast to access if media.metadata_title: From eadb14c444ed94517a2b185a5cde0cf98a2d3c81 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 04:05:59 -0400 Subject: [PATCH 169/454] Remove unused `json_serial` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 19424e8d..beb1b7b8 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -31,7 +31,7 @@ from common.logger import log from common.errors import ( NoFormatException, NoMediaException, NoMetadataException, DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, - json_serial, remove_enclosed, ) + remove_enclosed, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, delete_file, From 92474f57c39c8ee74f6f91dcd162702ea0cb5746 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 04:37:38 -0400 Subject: [PATCH 170/454] Switch to `has_metadata` property --- tubesync/sync/signals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 2b92f956..32b0b5f6 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -219,7 +219,7 @@ def media_post_save(sender, instance, created, **kwargs): if not existing_media_download_task: # Recalculate the "can_download" flag, this may # need to change if the source specifications have been changed - if instance.metadata: + if media.has_metadata: if instance.get_format_str(): if not instance.can_download: instance.can_download = True @@ -249,12 +249,12 @@ def media_post_save(sender, instance, created, **kwargs): ) # If the media is missing metadata schedule it to be downloaded - if not (instance.skip or instance.metadata or existing_media_metadata_task): + if not (media.skip or media.has_metadata or existing_media_metadata_task): log.info(f'Scheduling task to download metadata for: {instance.url}') - verbose_name = _('Downloading metadata for "{}"') + verbose_name = _('Downloading metadata for: {}: "{}"') download_media_metadata( str(instance.pk), - verbose_name=verbose_name.format(instance.pk), + verbose_name=verbose_name.format(media.key, media.name), ) # If the media is missing a thumbnail schedule it to be downloaded (unless we are skipping this media) if not instance.thumb_file_exists: From 34c8baa5efa31981c3349aae385b5b0bbfd4d38f Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 04:49:07 -0400 Subject: [PATCH 171/454] Add `metadata_clear` function --- tubesync/sync/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 000a59d3..17481f48 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1106,6 +1106,12 @@ class Media(models.Model): return self.metadata is not None + def metadata_clear(self, /, *, save=False): + self.metadata = None + if save: + self.save() + + def metadata_dumps(self, arg_dict=dict()): from common.utils import json_serial data = arg_dict or self.new_metadata.with_formats From b68b7132bd9b1678b07a0b746738e0dba95c89bb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 04:52:25 -0400 Subject: [PATCH 172/454] Switch to `metadata_clear` function --- tubesync/sync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index e951e56b..e7932a86 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -664,7 +664,7 @@ class MediaSkipView(FormView, SingleObjectMixin): for file in all_related_files: delete_file(file) # Reset all download data - self.object.metadata = None + self.object.metadata_clear() self.object.downloaded = False self.object.downloaded_audio_codec = None self.object.downloaded_video_codec = None From 52c69ef166f473cac1f6806238b67c4c342f8892 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 05:20:05 -0400 Subject: [PATCH 173/454] Switch to `timestamp_to_datetime` function --- tubesync/sync/management/commands/import-existing-media.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/import-existing-media.py b/tubesync/sync/management/commands/import-existing-media.py index 7dddc8c4..3813b497 100644 --- a/tubesync/sync/management/commands/import-existing-media.py +++ b/tubesync/sync/management/commands/import-existing-media.py @@ -2,6 +2,7 @@ import os from pathlib import Path from django.core.management.base import BaseCommand, CommandError from common.logger import log +from common.timestamp import timestamp_to_datetime from sync.choices import FileExtension from sync.models import Source, Media @@ -55,11 +56,11 @@ class Command(BaseCommand): item.downloaded = True item.downloaded_filesize = Path(filepath).stat().st_size # set a reasonable download date - date = item.metadata_published(Path(filepath).stat().st_mtime) + date = timestamp_to_datetime(Path(filepath).stat().st_mtime) if item.published and item.published > date: date = item.published if item.has_metadata: - metadata_date = item.metadata_published(item.get_metadata_first_value('epoch', 0)) + metadata_date = timestamp_to_datetime(item.get_metadata_first_value('epoch', 0)) if metadata_date and metadata_date > date: date = metadata_date if item.download_date and item.download_date > date: From 3ceedbde6fc8a03d0a670b169b0e11b095a6c7dc Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 05:28:11 -0400 Subject: [PATCH 174/454] Use a consistent verbose name for the metadata task --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index beb1b7b8..9edc2f38 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -349,10 +349,10 @@ def index_source_task(source_id): if new_media_instance: log.info(f'Indexed new media: {source} / {media}') log.info(f'Scheduling task to download metadata for: {media.url}') - verbose_name = _('Downloading metadata for "{}"') + verbose_name = _('Downloading metadata for: {}: "{}"') download_media_metadata( str(media.pk), - verbose_name=verbose_name.format(media.pk), + verbose_name=verbose_name.format(media.key, media.name), ) # Reset task.verbose_name to the saved value update_task_status(task, None) From a0354902f6192f770f7110643e920a5707526abf Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 05:44:04 -0400 Subject: [PATCH 175/454] Write new metadata to the table --- tubesync/sync/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 9edc2f38..eea2a750 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -494,7 +494,8 @@ def download_media_metadata(media_id): if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) media.ingest_metadata(response) - media.metadata = media.metadata_dumps(arg_dict=response) + pointer_dict = {'_using_table': True} + media.metadata = media.metadata_dumps(arg_dict=pointer_dict) upload_date = media.upload_date # Media must have a valid upload date if upload_date: From 23880bc51f3788d2b0d931d1ca2c3859e21b030a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 06:05:04 -0400 Subject: [PATCH 176/454] Save metadata to the table --- tubesync/sync/models.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 17481f48..26607d99 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1144,11 +1144,18 @@ class Media(models.Model): def save_to_metadata(self, key, value, /): data = self.loaded_metadata data[key] = value - #epoch = self.get_metadata_first_value('epoch', arg_dict=data) - #migrated = dict(migrated=True, epoch=epoch) - self.metadata = self.metadata_dumps(arg_dict=data) - self.save() self.ingest_metadata(data) + using_new_metadata = self.get_metadata_first_value( + ('migrated', '_using_table',), + False, + arg_dict=data, + ) + if not using_new_metadata: + epoch = self.get_metadata_first_value('epoch', arg_dict=data) + migrated = dict(migrated=True, epoch=epoch) + migrated['_using_table'] = True + self.metadata = self.metadata_dumps(arg_dict=migrated) + self.save() from common.logger import log log.debug(f'Saved to metadata: {self.key} / {self.uuid}: {key=}: {value}') From b3fd4ddae9fbb3d98ba815da878e064d7a053641 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 12:13:14 -0400 Subject: [PATCH 177/454] `published` was already not a naive `datetime` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index eea2a750..50f8ff16 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -502,7 +502,7 @@ def download_media_metadata(media_id): media.published = timezone.make_aware(upload_date) published = media.metadata_published() if published: - media.published = timezone.make_aware(published) + media.published = published # Store title in DB so it's fast to access if media.metadata_title: From 21d884c6e99b13d0c7ddb7cb1b9d8f3bcc745c82 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 13:02:17 -0400 Subject: [PATCH 178/454] Use the default locale before the new one is ready --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d68efe3e..2c1af46f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,9 +48,11 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va set -x && \ apt-get update && \ # Install locales + LC_ALL='C' LANG='C' LANGUAGE='C' \ apt-get -y --no-install-recommends install locales && \ - printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \ - locale-gen en_US.UTF-8 && \ + # localedef -v -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \ + printf -- "${LC_ALL} UTF-8\n" > /etc/locale.gen && \ + locale-gen && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ From 10e5a72e2c9679e716a36f965a8350246df0ea66 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 13:15:45 -0400 Subject: [PATCH 179/454] Match `--list-archive` output --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2c1af46f..29c5b0aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,9 @@ ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" \ APT_KEEP_ARCHIVES=1 \ HOME="/root" \ - LANGUAGE="en_US.UTF-8" \ - LANG="en_US.UTF-8" \ - LC_ALL="en_US.UTF-8" \ + LANGUAGE="en_US.utf8" \ + LANG="en_US.utf8" \ + LC_ALL="en_US.utf8" \ TERM="xterm" \ # Do not include compiled byte-code PIP_NO_COMPILE=1 \ @@ -51,7 +51,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va LC_ALL='C' LANG='C' LANGUAGE='C' \ apt-get -y --no-install-recommends install locales && \ # localedef -v -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \ - printf -- "${LC_ALL} UTF-8\n" > /etc/locale.gen && \ + printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \ locale-gen && \ # Clean up apt-get -y autopurge && \ From 2b27d1bd1e7e83c40aaeef23bea698f2c38bd854 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 13:34:58 -0400 Subject: [PATCH 180/454] Do not use the glibc alias --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 29c5b0aa..c21a18c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,9 @@ ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" \ APT_KEEP_ARCHIVES=1 \ HOME="/root" \ - LANGUAGE="en_US.utf8" \ - LANG="en_US.utf8" \ - LC_ALL="en_US.utf8" \ + LANGUAGE="en_US.UTF-8" \ + LANG="en_US.UTF-8" \ + LC_ALL="en_US.UTF-8" \ TERM="xterm" \ # Do not include compiled byte-code PIP_NO_COMPILE=1 \ @@ -48,7 +48,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va set -x && \ apt-get update && \ # Install locales - LC_ALL='C' LANG='C' LANGUAGE='C' \ + LC_ALL='C.UTF-8' LANG='C.UTF-8' LANGUAGE='C.UTF-8' \ apt-get -y --no-install-recommends install locales && \ # localedef -v -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && \ printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \ From d43dcb9f715ddda5777ed98f713b8f9490326d1d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 14:24:37 -0400 Subject: [PATCH 181/454] Add files via upload --- .../0031_squashed_metadata_metadataformat.py | 4 +- ...s_alter_metadataformat_options_and_more.py | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tubesync/sync/migrations/0032_alter_metadata_options_alter_metadataformat_options_and_more.py diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index 71e681ff..13189f10 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.8 on 2025-04-22 03:34 +# Generated by Django 5.1.8 on 2025-04-23 18:10 import django.db.models.deletion import sync.models @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): - replaces = [('sync', '0031_metadata_metadataformat')] + replaces = [('sync', '0031_metadata_metadataformat'), ('sync', '0032_alter_metadata_options_alter_metadataformat_options_and_more')] dependencies = [ ('sync', '0001_squashed_0030_alter_source_source_vcodec'), diff --git a/tubesync/sync/migrations/0032_alter_metadata_options_alter_metadataformat_options_and_more.py b/tubesync/sync/migrations/0032_alter_metadata_options_alter_metadataformat_options_and_more.py new file mode 100644 index 00000000..c6e00043 --- /dev/null +++ b/tubesync/sync/migrations/0032_alter_metadata_options_alter_metadataformat_options_and_more.py @@ -0,0 +1,78 @@ +# Generated by Django 5.1.8 on 2025-04-23 18:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0031_metadata_metadataformat'), + ] + + operations = [ + migrations.AlterModelOptions( + name='metadata', + options={'get_latest_by': ['-retrieved', '-created'], 'verbose_name': 'Metadata about Media', 'verbose_name_plural': 'Metadata about Media'}, + ), + migrations.AlterModelOptions( + name='metadataformat', + options={'ordering': ['site', 'key', 'number'], 'verbose_name': 'Format from Media Metadata', 'verbose_name_plural': 'Formats from Media Metadata'}, + ), + migrations.AlterUniqueTogether( + name='metadataformat', + unique_together={('metadata', 'site', 'key', 'number')}, + ), + migrations.AlterField( + model_name='metadata', + name='key', + field=models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key'), + ), + migrations.AlterField( + model_name='metadata', + name='media', + field=models.OneToOneField(help_text='Media the metadata belongs to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='new_metadata', to='sync.media'), + ), + migrations.AlterField( + model_name='metadata', + name='published', + field=models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published'), + ), + migrations.AlterField( + model_name='metadata', + name='site', + field=models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site'), + ), + migrations.AlterField( + model_name='metadata', + name='uploaded', + field=models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded'), + ), + migrations.AlterField( + model_name='metadataformat', + name='key', + field=models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key'), + ), + migrations.AlterField( + model_name='metadataformat', + name='metadata', + field=models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata'), + ), + migrations.AlterField( + model_name='metadataformat', + name='site', + field=models.CharField(blank=True, db_index=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site'), + ), + migrations.AlterModelTable( + name='metadata', + table='sync_media_metadata', + ), + migrations.AlterModelTable( + name='metadataformat', + table='sync_media_metadata_format', + ), + migrations.RemoveField( + model_name='metadataformat', + name='code', + ), + ] From ba22aa2a4711aa6dea0d6e08fb4d9ce53613ddd4 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 14:43:52 -0400 Subject: [PATCH 182/454] Keep the formats table clean --- tubesync/sync/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 26607d99..b982b094 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1850,6 +1850,8 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_formats(self, formats=list(), /): + # delete anything we won't reach + self.format.filter(number__gte=len(formats)).delete() for number, format in enumerate(formats, start=1): mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) mdf.value = format From 33acb5fc26f63356bfa8064297d2f183a54c331e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 15:19:06 -0400 Subject: [PATCH 183/454] Create 0032_metadata_transfer.py --- .../sync/migrations/0032_metadata_transfer.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tubesync/sync/migrations/0032_metadata_transfer.py diff --git a/tubesync/sync/migrations/0032_metadata_transfer.py b/tubesync/sync/migrations/0032_metadata_transfer.py new file mode 100644 index 00000000..ce4b8344 --- /dev/null +++ b/tubesync/sync/migrations/0032_metadata_transfer.py @@ -0,0 +1,36 @@ +# Hand-crafted data migration + +from django.db import migrations +from common.utils import django_queryset_generator as qs_gen + + +def use_tables(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + qs = Media.objects.filter(metadata__isnull=False) + for media in qs_gen(qs): + media.save_to_metadata('migrated', True) + +def restore_metadata_column(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + qs = Media.objects.filter(metadata__isnull=False) + for media in qs_gen(qs): + metadata = media.loaded_metadata + del metadata['migrated'] + del metadata['_using_table'] + media.metadata = media.metadata_dumps(arg_dict=metadata) + media.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0031_squashed_metadata_metadataformat'), + ] + + operations = [ + migrations.RunPython( + code=use_tables, + reverse_code=restore_metadata_column, + ), + ] + From 091b4cdaad4e6f26b05e8d50504f2e99580129bd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 15:42:02 -0400 Subject: [PATCH 184/454] Invalidate cache when clearing metadata --- tubesync/sync/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index b982b094..e585374b 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1108,6 +1108,7 @@ class Media(models.Model): def metadata_clear(self, /, *, save=False): self.metadata = None + setattr(self, '_cached_metadata_dict', None) if save: self.save() From 12037412350337c5c5949c82b35b727b6b61ade7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 16:08:34 -0400 Subject: [PATCH 185/454] Use a single counter --- tubesync/sync/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e585374b..797f1157 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1851,12 +1851,12 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_formats(self, formats=list(), /): - # delete anything we won't reach - self.format.filter(number__gte=len(formats)).delete() for number, format in enumerate(formats, start=1): mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) mdf.value = format mdf.save() + # delete any numbers we did not overwrite or create + self.format.filter(number__gt=number).delete() @property def with_formats(self): From 050b1579e2097cb53a970ccbe84f4dff00c4857a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:05:19 -0400 Subject: [PATCH 186/454] Add files via upload --- ...nitial_squashed_0010_auto_20210924_0554.py | 99 +++++++++++++++++++ ...1_1654_squashed_0020_auto_20231024_1825.py | 97 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py create mode 100644 tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py new file mode 100644 index 00000000..f356f8b1 --- /dev/null +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -0,0 +1,99 @@ +# Generated by Django 5.1.8 on 2025-04-24 01:54 + +import django.core.files.storage +import django.db.models.deletion +import sync.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Source', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created')), + ('last_crawl', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl')), + ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type')), + ('key', models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key')), + ('name', models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name')), + ('directory', models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory')), + ('index_schedule', models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule')), + ('delete_old_media', models.BooleanField(default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media')), + ('days_to_keep', models.PositiveSmallIntegerField(default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep')), + ('source_resolution', models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution')), + ('source_vcodec', models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec')), + ('source_acodec', models.CharField(choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec')), + ('prefer_60fps', models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps')), + ('prefer_hdr', models.BooleanField(default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr')), + ('fallback', models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback')), + ('has_failed', models.BooleanField(default=False, help_text='Source has failed to index media', verbose_name='has failed')), + ('copy_thumbnails', models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails')), + ('media_format', models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format')), + ('write_nfo', models.BooleanField(default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo')), + ('download_cap', models.IntegerField(choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap')), + ('download_media', models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media')), + ], + options={ + 'verbose_name': 'Source', + 'verbose_name_plural': 'Sources', + }, + ), + migrations.CreateModel( + name='MediaServer', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('server_type', models.CharField(choices=[('p', 'Plex')], db_index=True, default='p', help_text='Server type', max_length=1, verbose_name='server type')), + ('host', models.CharField(db_index=True, help_text='Hostname or IP address of the media server', max_length=200, verbose_name='host')), + ('port', models.PositiveIntegerField(db_index=True, help_text='Port number of the media server', verbose_name='port')), + ('use_https', models.BooleanField(default=True, help_text='Connect to the media server over HTTPS', verbose_name='use https')), + ('verify_https', models.BooleanField(default=False, help_text='If connecting over HTTPS, verify the SSL certificate is valid', verbose_name='verify https')), + ('options', models.TextField(blank=True, help_text='JSON encoded options for the media server', null=True, verbose_name='options')), + ], + options={ + 'verbose_name': 'Media Server', + 'verbose_name_plural': 'Media Servers', + 'unique_together': {('host', 'port')}, + }, + ), + migrations.CreateModel( + name='Media', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created')), + ('published', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published')), + ('key', models.CharField(db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key')), + ('thumb', models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width')), + ('thumb_width', models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width')), + ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height')), + ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')), + ('can_download', models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download')), + ('media_file', models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file')), + ('skip', models.BooleanField(db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip')), + ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')), + ('download_date', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date')), + ('downloaded_format', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded format')), + ('downloaded_height', models.PositiveIntegerField(blank=True, help_text='Height in pixels of the downloaded media', null=True, verbose_name='downloaded height')), + ('downloaded_width', models.PositiveIntegerField(blank=True, help_text='Width in pixels of the downloaded media', null=True, verbose_name='downloaded width')), + ('downloaded_audio_codec', models.CharField(blank=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec')), + ('downloaded_video_codec', models.CharField(blank=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec')), + ('downloaded_container', models.CharField(blank=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format')), + ('downloaded_fps', models.PositiveSmallIntegerField(blank=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps')), + ('downloaded_hdr', models.BooleanField(default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr')), + ('downloaded_filesize', models.PositiveBigIntegerField(blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize')), + ('source', models.ForeignKey(help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source')), + ], + options={ + 'verbose_name': 'Media', + 'verbose_name_plural': 'Media', + 'unique_together': {('source', 'key')}, + }, + ), + ] diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py new file mode 100644 index 00000000..acb1e384 --- /dev/null +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -0,0 +1,97 @@ +# Generated by Django 5.1.8 on 2025-04-24 01:55 + +import django.core.files.storage +import django.core.validators +import sync.fields +import sync.models +from django.db import migrations, models + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# sync.migrations.0013_fix_elative_media_file + +class Migration(migrations.Migration): + + replaces = [('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), ('sync', '0015_auto_20230213_0603'), ('sync', '0016_auto_20230214_2052'), ('sync', '0017_alter_source_sponsorblock_categories'), ('sync', '0018_source_subtitles'), ('sync', '0019_add_delete_removed_media'), ('sync', '0020_auto_20231024_1825')] + + dependencies = [ + ('sync', '0010_auto_20210924_0554'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_json', + field=models.BooleanField(default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'), + ), + migrations.AlterField( + model_name='media', + name='downloaded_format', + field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), + ), + migrations.RunPython( + code=sync.migrations.0013_fix_elative_media_file.fix_media_file, + ), + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location='/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AddField( + model_name='media', + name='manual_skip', + field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'), + ), + migrations.AlterField( + model_name='media', + name='skip', + field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), + ), + migrations.AddField( + model_name='source', + name='embed_metadata', + field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), + ), + migrations.AddField( + model_name='source', + name='embed_thumbnail', + field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), + ), + migrations.AddField( + model_name='source', + name='enable_sponsorblock', + field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), + ), + migrations.AddField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', max_length=128, possible_choices=('', ''), separator=''), + ), + migrations.AddField( + model_name='source', + name='write_subtitles', + field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), + ), + migrations.AddField( + model_name='source', + name='delete_removed_media', + field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), + ), + migrations.AddField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), + ), + migrations.AddField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), + ), + migrations.AddField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), + ), + ] From 7376e198c5f18557c29c915edcdff1bf812d19a5 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:06:53 -0400 Subject: [PATCH 187/454] Delete tubesync/sync/migrations/0020_auto_20231024_1825.py --- .../migrations/0020_auto_20231024_1825.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 tubesync/sync/migrations/0020_auto_20231024_1825.py diff --git a/tubesync/sync/migrations/0020_auto_20231024_1825.py b/tubesync/sync/migrations/0020_auto_20231024_1825.py deleted file mode 100644 index 295339a8..00000000 --- a/tubesync/sync/migrations/0020_auto_20231024_1825.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.22 on 2023-10-24 17:25 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0019_add_delete_removed_media'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), - ), - migrations.AlterField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), - ), - migrations.AlterField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), - ), - ] From 03ae41c1659b73fecff78d123310419400d34c50 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:07:37 -0400 Subject: [PATCH 188/454] Update 0021_source_copy_channel_images.py --- tubesync/sync/migrations/0021_source_copy_channel_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0021_source_copy_channel_images.py b/tubesync/sync/migrations/0021_source_copy_channel_images.py index 5d568925..fce1f5be 100644 --- a/tubesync/sync/migrations/0021_source_copy_channel_images.py +++ b/tubesync/sync/migrations/0021_source_copy_channel_images.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0020_auto_20231024_1825'), + ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ] operations = [ From 9c2648790d9464b027811dca0ab321e40444c8db Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:09:26 -0400 Subject: [PATCH 189/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- .../0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index acb1e384..16488a2c 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): replaces = [('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), ('sync', '0015_auto_20230213_0603'), ('sync', '0016_auto_20230214_2052'), ('sync', '0017_alter_source_sponsorblock_categories'), ('sync', '0018_source_subtitles'), ('sync', '0019_add_delete_removed_media'), ('sync', '0020_auto_20231024_1825')] dependencies = [ - ('sync', '0010_auto_20210924_0554'), + ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ] operations = [ From 3e47da7870c1584757a533ff874fad60a3805663 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:23:43 -0400 Subject: [PATCH 190/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- ...1_1654_squashed_0020_auto_20231024_1825.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index 16488a2c..5d66186b 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -11,6 +11,27 @@ from django.db import migrations, models # Move them and any dependencies into this file, then update the # RunPython operations to refer to the local versions: # sync.migrations.0013_fix_elative_media_file +from django.conf import settings +from pathlib import Path + +def fix_media_file(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + download_dir = str(settings.DOWNLOAD_ROOT) + download_dir_path = Path(download_dir) + for media in Media.objects.filter(downloaded=True): + if media.media_file.path.startswith(download_dir): + media_path = Path(media.media_file.path) + relative_path = media_path.relative_to(download_dir_path) + media.media_file.name = str(relative_path) + media.save() + +# Function above has been copied/modified and RunPython operations adjusted. + +def media_file_location(): + return str(settings.DOWNLOAD_ROOT) + +# Used the above function for storage location. + class Migration(migrations.Migration): @@ -32,12 +53,13 @@ class Migration(migrations.Migration): field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), ), migrations.RunPython( - code=sync.migrations.0013_fix_elative_media_file.fix_media_file, + code=fix_media_file, + reverse_code=migrations.RunPython.noop, ), migrations.AlterField( model_name='media', name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location='/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file'), ), migrations.AddField( model_name='media', From bd02f8e483b9f9b66418e874c98488b710f75900 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:29:59 -0400 Subject: [PATCH 191/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../0001_initial_squashed_0010_auto_20210924_0554.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index f356f8b1..462155f0 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -4,9 +4,14 @@ import django.core.files.storage import django.db.models.deletion import sync.models import uuid +from django.conf import settings from django.db import migrations, models +def media_file_location(): + return str(settings.DOWNLOAD_ROOT) + + class Migration(migrations.Migration): replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] @@ -75,7 +80,7 @@ class Migration(migrations.Migration): ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height')), ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')), ('can_download', models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download')), - ('media_file', models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file')), + ('media_file', models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file')), ('skip', models.BooleanField(db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip')), ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')), ('download_date', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date')), From 8dba1a184da543d08167f45e38e8a1cd57b35daf Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:52:52 -0400 Subject: [PATCH 192/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- .../0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index 5d66186b..05ee12da 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -89,7 +89,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='source', name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', max_length=128, possible_choices=('', ''), separator=''), + field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', max_length=128, possible_choices=('', ''), separator=','), ), migrations.AddField( model_name='source', From 86b38a05e7cdc15e80162ea83201a2cdf3f6db10 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:58:37 -0400 Subject: [PATCH 193/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../migrations/0001_squashed_0030_alter_source_source_vcodec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index ac4e87eb..b3373f02 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -39,6 +39,7 @@ class Migration(migrations.Migration): replaces = [ ('sync', '0001_initial'), + ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), @@ -49,6 +50,7 @@ class Migration(migrations.Migration): ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554'), ('sync', '0011_auto_20220201_1654'), + ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), From b797abae5392c0679447821f82994e566ea82b63 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 22:59:44 -0400 Subject: [PATCH 194/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../migrations/0001_initial_squashed_0010_auto_20210924_0554.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index 462155f0..ef406985 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -16,6 +16,8 @@ class Migration(migrations.Migration): replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] + initial = True + dependencies = [ ] From 57c39fcc74710c3cb765f8f6739ec215fd563a0b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:05:12 -0400 Subject: [PATCH 195/454] Rename tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py to tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/{migrations => }/0001_squashed_0030_alter_source_source_vcodec.py (100%) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py similarity index 100% rename from tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py rename to tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py From f220c797bc9af744f813774c29891cf5d8f3df2e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:07:56 -0400 Subject: [PATCH 196/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 00efa0f6..381dace4 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -9,7 +9,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0001_squashed_0030_alter_source_source_vcodec'), + ('sync', '0030_alter_source_source_vcodec'), ] operations = [ From 9803d5890fc7bf9bb84035ab549d53958449d8d1 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:10:16 -0400 Subject: [PATCH 197/454] Rename tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py to tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/{ => migrations}/0001_squashed_0030_alter_source_source_vcodec.py (100%) diff --git a/tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py similarity index 100% rename from tubesync/sync/0001_squashed_0030_alter_source_source_vcodec.py rename to tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py From bcb3ae021525e0032e599f4b8456bf202489ca9c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:13:03 -0400 Subject: [PATCH 198/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../migrations/0001_squashed_0030_alter_source_source_vcodec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index b3373f02..1183f3d8 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -72,7 +72,7 @@ class Migration(migrations.Migration): ('sync', '0030_alter_source_source_vcodec'), ] - initial = True + initial = False dependencies = [ ] From a5c8a3587587cf49075317979c9736485f9e03dd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:15:44 -0400 Subject: [PATCH 199/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 1183f3d8..9bdc03b9 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -39,7 +39,7 @@ class Migration(migrations.Migration): replaces = [ ('sync', '0001_initial'), - ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), + # ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), @@ -50,7 +50,7 @@ class Migration(migrations.Migration): ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554'), ('sync', '0011_auto_20220201_1654'), - ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + # ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), @@ -72,7 +72,7 @@ class Migration(migrations.Migration): ('sync', '0030_alter_source_source_vcodec'), ] - initial = False + initial = True dependencies = [ ] From a2f8e7074c3f7bde251cedee7124007d67546c3d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:19:54 -0400 Subject: [PATCH 200/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../migrations/0001_initial_squashed_0010_auto_20210924_0554.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index ef406985..bcee0775 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] - initial = True + initial = False dependencies = [ ] From 706a34d690699c80cf65ecccc6d8dab3c654e958 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:22:20 -0400 Subject: [PATCH 201/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../migrations/0001_initial_squashed_0010_auto_20210924_0554.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index bcee0775..ef406985 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] - initial = False + initial = True dependencies = [ ] From de26749409db329835850f05dae15337d793ae1d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:24:41 -0400 Subject: [PATCH 202/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- ...quashed_0030_alter_source_source_vcodec.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 9bdc03b9..cc8ae868 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -38,28 +38,28 @@ def media_file_location(): class Migration(migrations.Migration): replaces = [ - ('sync', '0001_initial'), - # ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), - ('sync', '0002_auto_20201213_0817'), - ('sync', '0003_source_copy_thumbnails'), - ('sync', '0004_source_media_format'), - ('sync', '0005_auto_20201219_0312'), - ('sync', '0006_source_write_nfo'), - ('sync', '0007_auto_20201219_0645'), - ('sync', '0008_source_download_cap'), - ('sync', '0009_auto_20210218_0442'), - ('sync', '0010_auto_20210924_0554'), - ('sync', '0011_auto_20220201_1654'), - # ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), - ('sync', '0012_alter_media_downloaded_format'), - ('sync', '0013_fix_elative_media_file'), - ('sync', '0014_alter_media_media_file'), - ('sync', '0015_auto_20230213_0603'), - ('sync', '0016_auto_20230214_2052'), - ('sync', '0017_alter_source_sponsorblock_categories'), - ('sync', '0018_source_subtitles'), - ('sync', '0019_add_delete_removed_media'), - ('sync', '0020_auto_20231024_1825'), + # ('sync', '0001_initial'), + ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), + # ('sync', '0002_auto_20201213_0817'), + # ('sync', '0003_source_copy_thumbnails'), + # ('sync', '0004_source_media_format'), + # ('sync', '0005_auto_20201219_0312'), + # ('sync', '0006_source_write_nfo'), + # ('sync', '0007_auto_20201219_0645'), + # ('sync', '0008_source_download_cap'), + # ('sync', '0009_auto_20210218_0442'), + # ('sync', '0010_auto_20210924_0554'), + # ('sync', '0011_auto_20220201_1654'), + ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + # ('sync', '0012_alter_media_downloaded_format'), + # ('sync', '0013_fix_elative_media_file'), + # ('sync', '0014_alter_media_media_file'), + # ('sync', '0015_auto_20230213_0603'), + # ('sync', '0016_auto_20230214_2052'), + # ('sync', '0017_alter_source_sponsorblock_categories'), + # ('sync', '0018_source_subtitles'), + # ('sync', '0019_add_delete_removed_media'), + # ('sync', '0020_auto_20231024_1825'), ('sync', '0021_source_copy_channel_images'), ('sync', '0022_add_delete_files_on_disk'), ('sync', '0023_media_duration_filter'), From 0152c971c99be82fc53141766ff7fd9fecc56363 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:54:45 -0400 Subject: [PATCH 203/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../migrations/0001_initial_squashed_0010_auto_20210924_0554.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index ef406985..2c6bfa4a 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -14,7 +14,7 @@ def media_file_location(): class Migration(migrations.Migration): - replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] + # replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] initial = True From ca2e2c8ac5a60044a1215456d543ced90a3852ab Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Apr 2025 23:55:47 -0400 Subject: [PATCH 204/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- .../0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index 05ee12da..6b4c6d1c 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -35,7 +35,7 @@ def media_file_location(): class Migration(migrations.Migration): - replaces = [('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), ('sync', '0015_auto_20230213_0603'), ('sync', '0016_auto_20230214_2052'), ('sync', '0017_alter_source_sponsorblock_categories'), ('sync', '0018_source_subtitles'), ('sync', '0019_add_delete_removed_media'), ('sync', '0020_auto_20231024_1825')] + # replaces = [('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), ('sync', '0015_auto_20230213_0603'), ('sync', '0016_auto_20230214_2052'), ('sync', '0017_alter_source_sponsorblock_categories'), ('sync', '0018_source_subtitles'), ('sync', '0019_add_delete_removed_media'), ('sync', '0020_auto_20231024_1825')] dependencies = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), From 79e7d885280a5ff673ce5aa47fa3ed08bd81873b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 00:00:16 -0400 Subject: [PATCH 205/454] Update 0001_initial_squashed_0010_auto_20210924_0554.py --- .../migrations/0001_initial_squashed_0010_auto_20210924_0554.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py index 2c6bfa4a..16ab1018 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py @@ -14,8 +14,6 @@ def media_file_location(): class Migration(migrations.Migration): - # replaces = [('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), ('sync', '0004_source_media_format'), ('sync', '0005_auto_20201219_0312'), ('sync', '0006_source_write_nfo'), ('sync', '0007_auto_20201219_0645'), ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554')] - initial = True dependencies = [ From d30b399301b1630b6baf57732b1ff28f5d58ca1e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 00:01:46 -0400 Subject: [PATCH 206/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- .../0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index 6b4c6d1c..d37b749d 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -35,8 +35,6 @@ def media_file_location(): class Migration(migrations.Migration): - # replaces = [('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), ('sync', '0014_alter_media_media_file'), ('sync', '0015_auto_20230213_0603'), ('sync', '0016_auto_20230214_2052'), ('sync', '0017_alter_source_sponsorblock_categories'), ('sync', '0018_source_subtitles'), ('sync', '0019_add_delete_removed_media'), ('sync', '0020_auto_20231024_1825')] - dependencies = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ] From fc0ab058a6687bcce6534759b24c453764d99dea Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 00:05:08 -0400 Subject: [PATCH 207/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 381dace4..00efa0f6 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -9,7 +9,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0030_alter_source_source_vcodec'), + ('sync', '0001_squashed_0030_alter_source_source_vcodec'), ] operations = [ From 6d299a666178a2c8b69166eec9093d0e1d41f3fd Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 00:08:11 -0400 Subject: [PATCH 208/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- ...quashed_0030_alter_source_source_vcodec.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index cc8ae868..000bfc89 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -38,28 +38,8 @@ def media_file_location(): class Migration(migrations.Migration): replaces = [ - # ('sync', '0001_initial'), ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), - # ('sync', '0002_auto_20201213_0817'), - # ('sync', '0003_source_copy_thumbnails'), - # ('sync', '0004_source_media_format'), - # ('sync', '0005_auto_20201219_0312'), - # ('sync', '0006_source_write_nfo'), - # ('sync', '0007_auto_20201219_0645'), - # ('sync', '0008_source_download_cap'), - # ('sync', '0009_auto_20210218_0442'), - # ('sync', '0010_auto_20210924_0554'), - # ('sync', '0011_auto_20220201_1654'), ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), - # ('sync', '0012_alter_media_downloaded_format'), - # ('sync', '0013_fix_elative_media_file'), - # ('sync', '0014_alter_media_media_file'), - # ('sync', '0015_auto_20230213_0603'), - # ('sync', '0016_auto_20230214_2052'), - # ('sync', '0017_alter_source_sponsorblock_categories'), - # ('sync', '0018_source_subtitles'), - # ('sync', '0019_add_delete_removed_media'), - # ('sync', '0020_auto_20231024_1825'), ('sync', '0021_source_copy_channel_images'), ('sync', '0022_add_delete_files_on_disk'), ('sync', '0023_media_duration_filter'), From 8afe97c4d160a3f67d130bf7a75dde2ca2a68c32 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 13:23:00 -0400 Subject: [PATCH 209/454] Update 0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py --- ...0220201_1654_squashed_0020_auto_20231024_1825.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py index d37b749d..11d1268b 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py @@ -35,6 +35,19 @@ def media_file_location(): class Migration(migrations.Migration): + replaces = [ + ('sync', '0011_auto_20220201_1654'), + ('sync', '0012_alter_media_downloaded_format'), + ('sync', '0013_fix_elative_media_file'), + ('sync', '0014_alter_media_media_file'), + ('sync', '0015_auto_20230213_0603'), + ('sync', '0016_auto_20230214_2052'), + ('sync', '0017_alter_source_sponsorblock_categories'), + ('sync', '0018_source_subtitles'), + ('sync', '0019_add_delete_removed_media'), + ('sync', '0020_auto_20231024_1825'), + ] + dependencies = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ] From ff0f72b884330c9f486db319e9c641dc378ea165 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 13:31:25 -0400 Subject: [PATCH 210/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 000bfc89..983f2e39 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -39,7 +39,17 @@ class Migration(migrations.Migration): replaces = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), - ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + # ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + ('sync', '0011_auto_20220201_1654'), + ('sync', '0012_alter_media_downloaded_format'), + ('sync', '0013_fix_elative_media_file'), + ('sync', '0014_alter_media_media_file'), + ('sync', '0015_auto_20230213_0603'), + ('sync', '0016_auto_20230214_2052'), + ('sync', '0017_alter_source_sponsorblock_categories'), + ('sync', '0018_source_subtitles'), + ('sync', '0019_add_delete_removed_media'), + ('sync', '0020_auto_20231024_1825'), ('sync', '0021_source_copy_channel_images'), ('sync', '0022_add_delete_files_on_disk'), ('sync', '0023_media_duration_filter'), From 6790456e0f79e607f544a0f3c09237e30e898d5e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 13:33:55 -0400 Subject: [PATCH 211/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../migrations/0001_squashed_0030_alter_source_source_vcodec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 983f2e39..666bb1dc 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -39,7 +39,7 @@ class Migration(migrations.Migration): replaces = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), - # ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), From 1ef1a5a985cdbe0e9146cb8fcb683e453c3ccc66 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 13:40:16 -0400 Subject: [PATCH 212/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 666bb1dc..3779e2bd 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -39,6 +39,16 @@ class Migration(migrations.Migration): replaces = [ ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), + ('sync', '0001_initial'), + ('sync', '0002_auto_20201213_0817'), + ('sync', '0003_source_copy_thumbnails'), + ('sync', '0004_source_media_format'), + ('sync', '0005_auto_20201219_0312'), + ('sync', '0006_source_write_nfo'), + ('sync', '0007_auto_20201219_0645'), + ('sync', '0008_source_download_cap'), + ('sync', '0009_auto_20210218_0442'), + ('sync', '0010_auto_20210924_0554'), ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), From ef4f4eed03e8037ffcb4f444743c5b119dcad81c Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 13:44:13 -0400 Subject: [PATCH 213/454] Update 0001_squashed_0030_alter_source_source_vcodec.py --- .../0001_squashed_0030_alter_source_source_vcodec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py index 3779e2bd..a9ea24e6 100644 --- a/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py +++ b/tubesync/sync/migrations/0001_squashed_0030_alter_source_source_vcodec.py @@ -38,7 +38,7 @@ def media_file_location(): class Migration(migrations.Migration): replaces = [ - ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), + # ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), ('sync', '0001_initial'), ('sync', '0002_auto_20201213_0817'), ('sync', '0003_source_copy_thumbnails'), @@ -49,7 +49,7 @@ class Migration(migrations.Migration): ('sync', '0008_source_download_cap'), ('sync', '0009_auto_20210218_0442'), ('sync', '0010_auto_20210924_0554'), - ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + # ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), ('sync', '0011_auto_20220201_1654'), ('sync', '0012_alter_media_downloaded_format'), ('sync', '0013_fix_elative_media_file'), From 6e97a24ec0a5882fe99f6636558a707d87a65a6a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 24 Apr 2025 15:09:11 -0400 Subject: [PATCH 214/454] Restore all the individual files --- ..._auto_20210924_0554.py => 0001_initial.py} | 20 +-- .../migrations/0002_auto_20201213_0817.py | 20 +++ .../migrations/0003_source_copy_thumbnails.py | 18 +++ .../migrations/0004_source_media_format.py | 18 +++ .../migrations/0005_auto_20201219_0312.py | 18 +++ .../sync/migrations/0006_source_write_nfo.py | 18 +++ .../migrations/0007_auto_20201219_0645.py | 18 +++ .../migrations/0008_source_download_cap.py | 18 +++ .../migrations/0009_auto_20210218_0442.py | 30 ++++ .../migrations/0010_auto_20210924_0554.py | 30 ++++ .../migrations/0011_auto_20220201_1654.py | 21 +++ ...1_1654_squashed_0020_auto_20231024_1825.py | 130 ------------------ .../0012_alter_media_downloaded_format.py | 18 +++ .../migrations/0013_fix_elative_media_file.py | 25 ++++ .../migrations/0014_alter_media_media_file.py | 21 +++ .../migrations/0015_auto_20230213_0603.py | 23 ++++ .../migrations/0016_auto_20230214_2052.py | 34 +++++ ...17_alter_source_sponsorblock_categories.py | 19 +++ .../sync/migrations/0018_source_subtitles.py | 27 ++++ .../0019_add_delete_removed_media.py | 17 +++ .../migrations/0020_auto_20231024_1825.py | 29 ++++ .../0021_source_copy_channel_images.py | 2 +- 22 files changed, 428 insertions(+), 146 deletions(-) rename tubesync/sync/migrations/{0001_initial_squashed_0010_auto_20210924_0554.py => 0001_initial.py} (82%) create mode 100644 tubesync/sync/migrations/0002_auto_20201213_0817.py create mode 100644 tubesync/sync/migrations/0003_source_copy_thumbnails.py create mode 100644 tubesync/sync/migrations/0004_source_media_format.py create mode 100644 tubesync/sync/migrations/0005_auto_20201219_0312.py create mode 100644 tubesync/sync/migrations/0006_source_write_nfo.py create mode 100644 tubesync/sync/migrations/0007_auto_20201219_0645.py create mode 100644 tubesync/sync/migrations/0008_source_download_cap.py create mode 100644 tubesync/sync/migrations/0009_auto_20210218_0442.py create mode 100644 tubesync/sync/migrations/0010_auto_20210924_0554.py create mode 100644 tubesync/sync/migrations/0011_auto_20220201_1654.py delete mode 100644 tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py create mode 100644 tubesync/sync/migrations/0012_alter_media_downloaded_format.py create mode 100644 tubesync/sync/migrations/0013_fix_elative_media_file.py create mode 100644 tubesync/sync/migrations/0014_alter_media_media_file.py create mode 100644 tubesync/sync/migrations/0015_auto_20230213_0603.py create mode 100644 tubesync/sync/migrations/0016_auto_20230214_2052.py create mode 100644 tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py create mode 100644 tubesync/sync/migrations/0018_source_subtitles.py create mode 100644 tubesync/sync/migrations/0019_add_delete_removed_media.py create mode 100644 tubesync/sync/migrations/0020_auto_20231024_1825.py diff --git a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py b/tubesync/sync/migrations/0001_initial.py similarity index 82% rename from tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py rename to tubesync/sync/migrations/0001_initial.py index 16ab1018..aa267a9a 100644 --- a/tubesync/sync/migrations/0001_initial_squashed_0010_auto_20210924_0554.py +++ b/tubesync/sync/migrations/0001_initial.py @@ -1,15 +1,10 @@ -# Generated by Django 5.1.8 on 2025-04-24 01:54 +# Generated by Django 3.1.4 on 2020-12-12 09:13 import django.core.files.storage +from django.db import migrations, models import django.db.models.deletion import sync.models import uuid -from django.conf import settings -from django.db import migrations, models - - -def media_file_location(): - return str(settings.DOWNLOAD_ROOT) class Migration(migrations.Migration): @@ -26,11 +21,11 @@ class Migration(migrations.Migration): ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid')), ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created')), ('last_crawl', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl')), - ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type')), + ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type')), ('key', models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key')), ('name', models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name')), ('directory', models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory')), - ('index_schedule', models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule')), + ('index_schedule', models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours')], db_index=True, default=21600, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule')), ('delete_old_media', models.BooleanField(default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media')), ('days_to_keep', models.PositiveSmallIntegerField(default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep')), ('source_resolution', models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution')), @@ -40,11 +35,6 @@ class Migration(migrations.Migration): ('prefer_hdr', models.BooleanField(default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr')), ('fallback', models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback')), ('has_failed', models.BooleanField(default=False, help_text='Source has failed to index media', verbose_name='has failed')), - ('copy_thumbnails', models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails')), - ('media_format', models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format')), - ('write_nfo', models.BooleanField(default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo')), - ('download_cap', models.IntegerField(choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap')), - ('download_media', models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media')), ], options={ 'verbose_name': 'Source', @@ -80,7 +70,7 @@ class Migration(migrations.Migration): ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height')), ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')), ('can_download', models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download')), - ('media_file', models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file')), + ('media_file', models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file')), ('skip', models.BooleanField(db_index=True, default=False, help_text='Media will be skipped and not downloaded', verbose_name='skip')), ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')), ('download_date', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the download completed', null=True, verbose_name='download date')), diff --git a/tubesync/sync/migrations/0002_auto_20201213_0817.py b/tubesync/sync/migrations/0002_auto_20201213_0817.py new file mode 100644 index 00000000..ab2f71e3 --- /dev/null +++ b/tubesync/sync/migrations/0002_auto_20201213_0817.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.4 on 2020-12-13 08:17 + +import django.core.files.storage +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + ] diff --git a/tubesync/sync/migrations/0003_source_copy_thumbnails.py b/tubesync/sync/migrations/0003_source_copy_thumbnails.py new file mode 100644 index 00000000..7bdbfdbc --- /dev/null +++ b/tubesync/sync/migrations/0003_source_copy_thumbnails.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-18 01:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0002_auto_20201213_0817'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='copy_thumbnails', + field=models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails'), + ), + ] diff --git a/tubesync/sync/migrations/0004_source_media_format.py b/tubesync/sync/migrations/0004_source_media_format.py new file mode 100644 index 00000000..f79a7036 --- /dev/null +++ b/tubesync/sync/migrations/0004_source_media_format.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-18 01:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0003_source_copy_thumbnails'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='media_format', + field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files', max_length=200, verbose_name='media format'), + ), + ] diff --git a/tubesync/sync/migrations/0005_auto_20201219_0312.py b/tubesync/sync/migrations/0005_auto_20201219_0312.py new file mode 100644 index 00000000..11ee8e27 --- /dev/null +++ b/tubesync/sync/migrations/0005_auto_20201219_0312.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 03:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0004_source_media_format'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_type', + field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'), + ), + ] diff --git a/tubesync/sync/migrations/0006_source_write_nfo.py b/tubesync/sync/migrations/0006_source_write_nfo.py new file mode 100644 index 00000000..d5fe9365 --- /dev/null +++ b/tubesync/sync/migrations/0006_source_write_nfo.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 03:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0005_auto_20201219_0312'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_nfo', + field=models.BooleanField(default=False, help_text='Write an NFO file with the media, these may be detected and used by some media servers', verbose_name='write nfo'), + ), + ] diff --git a/tubesync/sync/migrations/0007_auto_20201219_0645.py b/tubesync/sync/migrations/0007_auto_20201219_0645.py new file mode 100644 index 00000000..c757d679 --- /dev/null +++ b/tubesync/sync/migrations/0007_auto_20201219_0645.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 06:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0006_source_write_nfo'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='write_nfo', + field=models.BooleanField(default=False, help_text='Write an NFO file in XML with the media info, these may be detected and used by some media servers', verbose_name='write nfo'), + ), + ] diff --git a/tubesync/sync/migrations/0008_source_download_cap.py b/tubesync/sync/migrations/0008_source_download_cap.py new file mode 100644 index 00000000..4b3ec19f --- /dev/null +++ b/tubesync/sync/migrations/0008_source_download_cap.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 06:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0007_auto_20201219_0645'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='download_cap', + field=models.IntegerField(choices=[(0, 'No cap'), (604800, '1 week (7 days)'), (2592000, '1 month (30 days)'), (7776000, '3 months (90 days)'), (15552000, '6 months (180 days)'), (31536000, '1 year (365 days)'), (63072000, '2 years (730 days)'), (94608000, '3 years (1095 days)'), (157680000, '5 years (1825 days)'), (315360000, '10 years (3650 days)')], default=0, help_text='Do not download media older than this capped date', verbose_name='download cap'), + ), + ] diff --git a/tubesync/sync/migrations/0009_auto_20210218_0442.py b/tubesync/sync/migrations/0009_auto_20210218_0442.py new file mode 100644 index 00000000..45b94500 --- /dev/null +++ b/tubesync/sync/migrations/0009_auto_20210218_0442.py @@ -0,0 +1,30 @@ +# Generated by Django 3.1.6 on 2021-02-18 04:42 + +import django.core.files.storage +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0008_source_download_cap'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='download_media', + field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'), + ), + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AlterField( + model_name='source', + name='media_format', + field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), + ), + ] diff --git a/tubesync/sync/migrations/0010_auto_20210924_0554.py b/tubesync/sync/migrations/0010_auto_20210924_0554.py new file mode 100644 index 00000000..00160610 --- /dev/null +++ b/tubesync/sync/migrations/0010_auto_20210924_0554.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.7 on 2021-09-24 05:54 + +import django.core.files.storage +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0009_auto_20210218_0442'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AlterField( + model_name='source', + name='index_schedule', + field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'), + ), + migrations.AlterField( + model_name='source', + name='media_format', + field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'), + ), + ] diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py new file mode 100644 index 00000000..96d9f4a7 --- /dev/null +++ b/tubesync/sync/migrations/0011_auto_20220201_1654.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.11 on 2022-02-01 16:54 + +import django.core.files.storage +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0010_auto_20210924_0554'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_json', + field=models.BooleanField( + default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'), + ), + ] diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py b/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py deleted file mode 100644 index 11d1268b..00000000 --- a/tubesync/sync/migrations/0011_auto_20220201_1654_squashed_0020_auto_20231024_1825.py +++ /dev/null @@ -1,130 +0,0 @@ -# Generated by Django 5.1.8 on 2025-04-24 01:55 - -import django.core.files.storage -import django.core.validators -import sync.fields -import sync.models -from django.db import migrations, models - - -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# sync.migrations.0013_fix_elative_media_file -from django.conf import settings -from pathlib import Path - -def fix_media_file(apps, schema_editor): - Media = apps.get_model('sync', 'Media') - download_dir = str(settings.DOWNLOAD_ROOT) - download_dir_path = Path(download_dir) - for media in Media.objects.filter(downloaded=True): - if media.media_file.path.startswith(download_dir): - media_path = Path(media.media_file.path) - relative_path = media_path.relative_to(download_dir_path) - media.media_file.name = str(relative_path) - media.save() - -# Function above has been copied/modified and RunPython operations adjusted. - -def media_file_location(): - return str(settings.DOWNLOAD_ROOT) - -# Used the above function for storage location. - - -class Migration(migrations.Migration): - - replaces = [ - ('sync', '0011_auto_20220201_1654'), - ('sync', '0012_alter_media_downloaded_format'), - ('sync', '0013_fix_elative_media_file'), - ('sync', '0014_alter_media_media_file'), - ('sync', '0015_auto_20230213_0603'), - ('sync', '0016_auto_20230214_2052'), - ('sync', '0017_alter_source_sponsorblock_categories'), - ('sync', '0018_source_subtitles'), - ('sync', '0019_add_delete_removed_media'), - ('sync', '0020_auto_20231024_1825'), - ] - - dependencies = [ - ('sync', '0001_initial_squashed_0010_auto_20210924_0554'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='write_json', - field=models.BooleanField(default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'), - ), - migrations.AlterField( - model_name='media', - name='downloaded_format', - field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), - ), - migrations.RunPython( - code=fix_media_file, - reverse_code=migrations.RunPython.noop, - ), - migrations.AlterField( - model_name='media', - name='media_file', - field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=media_file_location()), upload_to=sync.models.get_media_file_path, verbose_name='media file'), - ), - migrations.AddField( - model_name='media', - name='manual_skip', - field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'), - ), - migrations.AlterField( - model_name='media', - name='skip', - field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), - ), - migrations.AddField( - model_name='source', - name='embed_metadata', - field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), - ), - migrations.AddField( - model_name='source', - name='embed_thumbnail', - field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), - ), - migrations.AddField( - model_name='source', - name='enable_sponsorblock', - field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), - ), - migrations.AddField( - model_name='source', - name='sponsorblock_categories', - field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', max_length=128, possible_choices=('', ''), separator=','), - ), - migrations.AddField( - model_name='source', - name='write_subtitles', - field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), - ), - migrations.AddField( - model_name='source', - name='delete_removed_media', - field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), - ), - migrations.AddField( - model_name='source', - name='filter_text', - field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), - ), - migrations.AddField( - model_name='source', - name='auto_subtitles', - field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), - ), - migrations.AddField( - model_name='source', - name='sub_langs', - field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), - ), - ] diff --git a/tubesync/sync/migrations/0012_alter_media_downloaded_format.py b/tubesync/sync/migrations/0012_alter_media_downloaded_format.py new file mode 100644 index 00000000..3e733efb --- /dev/null +++ b/tubesync/sync/migrations/0012_alter_media_downloaded_format.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-06 06:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0011_auto_20220201_1654'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='downloaded_format', + field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'), + ), + ] diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py new file mode 100644 index 00000000..c9eee22e --- /dev/null +++ b/tubesync/sync/migrations/0013_fix_elative_media_file.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.12 on 2022-04-06 06:19 + +from django.conf import settings +from django.db import migrations, models + + +def fix_media_file(apps, schema_editor): + Media = apps.get_model('sync', 'Media') + for media in Media.objects.filter(downloaded=True): + download_dir = str(settings.DOWNLOAD_ROOT) + + if media.media_file.name.startswith(download_dir): + media.media_file.name = media.media_file.name[len(download_dir) + 1:] + media.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0012_alter_media_downloaded_format'), + ] + + operations = [ + migrations.RunPython(fix_media_file) + ] diff --git a/tubesync/sync/migrations/0014_alter_media_media_file.py b/tubesync/sync/migrations/0014_alter_media_media_file.py new file mode 100644 index 00000000..530c8e81 --- /dev/null +++ b/tubesync/sync/migrations/0014_alter_media_media_file.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.15 on 2022-12-28 20:33 + +import django.core.files.storage +from django.conf import settings +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0013_fix_elative_media_file'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + ] diff --git a/tubesync/sync/migrations/0015_auto_20230213_0603.py b/tubesync/sync/migrations/0015_auto_20230213_0603.py new file mode 100644 index 00000000..54592f9d --- /dev/null +++ b/tubesync/sync/migrations/0015_auto_20230213_0603.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.17 on 2023-02-13 06:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0014_alter_media_media_file'), + ] + + operations = [ + migrations.AddField( + model_name='media', + name='manual_skip', + field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'), + ), + migrations.AlterField( + model_name='media', + name='skip', + field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'), + ), + ] diff --git a/tubesync/sync/migrations/0016_auto_20230214_2052.py b/tubesync/sync/migrations/0016_auto_20230214_2052.py new file mode 100644 index 00000000..ffba1952 --- /dev/null +++ b/tubesync/sync/migrations/0016_auto_20230214_2052.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.18 on 2023-02-14 20:52 + +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0015_auto_20230213_0603'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='embed_metadata', + field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), + ), + migrations.AddField( + model_name='source', + name='embed_thumbnail', + field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), + ), + migrations.AddField( + model_name='source', + name='enable_sponsorblock', + field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), + ), + migrations.AddField( + model_name='source', + name='sponsorblock_categories', + field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), + ), + ] diff --git a/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py b/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py new file mode 100644 index 00000000..cc9d9578 --- /dev/null +++ b/tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.18 on 2023-02-20 02:23 + +from django.db import migrations +import sync.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0016_auto_20230214_2052'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''), + ), + ] diff --git a/tubesync/sync/migrations/0018_source_subtitles.py b/tubesync/sync/migrations/0018_source_subtitles.py new file mode 100644 index 00000000..c526b994 --- /dev/null +++ b/tubesync/sync/migrations/0018_source_subtitles.py @@ -0,0 +1,27 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0017_alter_source_sponsorblock_categories'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_subtitles', + field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), + ), + migrations.AddField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'), + ), + migrations.AddField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30), + ), + ] diff --git a/tubesync/sync/migrations/0019_add_delete_removed_media.py b/tubesync/sync/migrations/0019_add_delete_removed_media.py new file mode 100644 index 00000000..0762be87 --- /dev/null +++ b/tubesync/sync/migrations/0019_add_delete_removed_media.py @@ -0,0 +1,17 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0018_source_subtitles'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='delete_removed_media', + field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'), + ), + ] diff --git a/tubesync/sync/migrations/0020_auto_20231024_1825.py b/tubesync/sync/migrations/0020_auto_20231024_1825.py new file mode 100644 index 00000000..295339a8 --- /dev/null +++ b/tubesync/sync/migrations/0020_auto_20231024_1825.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.22 on 2023-10-24 17:25 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0019_add_delete_removed_media'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='filter_text', + field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'), + ), + migrations.AlterField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'), + ), + migrations.AlterField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'), + ), + ] diff --git a/tubesync/sync/migrations/0021_source_copy_channel_images.py b/tubesync/sync/migrations/0021_source_copy_channel_images.py index fce1f5be..5d568925 100644 --- a/tubesync/sync/migrations/0021_source_copy_channel_images.py +++ b/tubesync/sync/migrations/0021_source_copy_channel_images.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0011_auto_20220201_1654_squashed_0020_auto_20231024_1825'), + ('sync', '0020_auto_20231024_1825'), ] operations = [ From c7b9efbaac9a3b91d2d8b64e3f7899b837f3f900 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 26 Apr 2025 07:13:31 -0400 Subject: [PATCH 215/454] Expand resolution matching to include height Return the requested codec before matching the first available format --- tubesync/sync/matching.py | 85 ++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 93f7e4d0..4196a9f8 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -121,17 +121,20 @@ def get_best_video_format(media): return False, False video_formats = multi_key_sort(video_formats, sort_keys, True) source_resolution = media.source.source_resolution.strip().upper() + source_resolution_height = media.source.source_resolution_height source_vcodec = media.source.source_vcodec exact_match, best_match = None, None - for fmt in video_formats: - # format_note was blank, match height instead - if '' == fmt['format'] and fmt['height'] == media.source.source_resolution_height: - fmt['format'] = source_resolution + def matched_resolution(fmt): + if fmt['format'] == source_resolution: + return True + elif fmt['height'] == source_resolution_height: + return True + return False # Of our filtered video formats, check for resolution + codec + hdr + fps match if media.source.prefer_60fps and media.source.prefer_hdr: for fmt in video_formats: # Check for an exact match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and fmt['is_hdr'] and fmt['is_60fps']): @@ -142,7 +145,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for a resolution, hdr and fps match but drop the codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and fmt['is_hdr'] and fmt['is_60fps']): # Close match exact_match, best_match = False, fmt @@ -158,7 +161,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution, codec and 60fps match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and fmt['is_60fps']): exact_match, best_match = False, fmt @@ -166,21 +169,21 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution and hdr match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and fmt['is_hdr']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution and 60fps match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and fmt['is_60fps']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution, codec and hdr match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and fmt['is_hdr']): exact_match, best_match = False, fmt @@ -188,14 +191,20 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution and codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution - if source_resolution == fmt['format']: + if matched_resolution(fmt): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for codec + if (source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: @@ -205,7 +214,7 @@ def get_best_video_format(media): if media.source.prefer_60fps and not media.source.prefer_hdr: for fmt in video_formats: # Check for an exact match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and fmt['is_60fps'] and not fmt['is_hdr']): @@ -216,7 +225,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for a resolution and fps match but drop the codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and fmt['is_60fps'] and not fmt['is_hdr']): exact_match, best_match = False, fmt @@ -239,7 +248,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for codec and resolution match but drop 60fps - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and not fmt['is_hdr']): exact_match, best_match = False, fmt @@ -247,14 +256,20 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for codec and resolution match only - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution - if source_resolution == fmt['format']: + if matched_resolution(fmt): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for codec + if (source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: @@ -264,7 +279,7 @@ def get_best_video_format(media): elif media.source.prefer_hdr and not media.source.prefer_60fps: for fmt in video_formats: # Check for an exact match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and fmt['is_hdr']): # Exact match @@ -274,7 +289,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for a resolution and fps match but drop the codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and fmt['is_hdr'] and not fmt['is_60fps']): exact_match, best_match = False, fmt @@ -297,7 +312,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for codec and resolution match but drop hdr - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and not fmt['is_60fps']): exact_match, best_match = False, fmt @@ -305,14 +320,20 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for codec and resolution match only - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution - if source_resolution == fmt['format']: + if matched_resolution(fmt): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for codec + if (source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: @@ -322,7 +343,7 @@ def get_best_video_format(media): elif not media.source.prefer_hdr and not media.source.prefer_60fps: for fmt in video_formats: # Check for an exact match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and not fmt['is_60fps'] and not fmt['is_hdr']): @@ -333,7 +354,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for a resolution, hdr and fps match but drop the codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and not fmt['is_hdr'] and not fmt['is_60fps']): # Close match exact_match, best_match = False, fmt @@ -349,7 +370,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution, codec and hdr match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and not fmt['is_hdr']): exact_match, best_match = False, fmt @@ -357,7 +378,7 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution, codec and 60fps match - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec'] and not fmt['is_60fps']): exact_match, best_match = False, fmt @@ -365,21 +386,27 @@ def get_best_video_format(media): if not best_match: for fmt in video_formats: # Check for resolution and codec - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution and not hdr - if (source_resolution == fmt['format'] and + if (matched_resolution(fmt) and not fmt['is_hdr']): exact_match, best_match = False, fmt break if not best_match: for fmt in video_formats: # Check for resolution - if source_resolution == fmt['format']: + if matched_resolution(fmt): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for codec + if (source_vcodec == fmt['vcodec']): exact_match, best_match = False, fmt break if not best_match: From 9f56c381660b6f6c564e3683ed25e4f2f7239402 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 26 Apr 2025 21:23:34 -0400 Subject: [PATCH 216/454] Be extra careful about which formats are deleted --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 797f1157..f36695c5 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1856,7 +1856,7 @@ class Metadata(models.Model): mdf.value = format mdf.save() # delete any numbers we did not overwrite or create - self.format.filter(number__gt=number).delete() + self.format.filter(site=self.site, key=self.key, number__gt=number).delete() @property def with_formats(self): From cbcd1185af14dd6365cb6b3e4ce8868965c90acc Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 26 Apr 2025 21:58:26 -0400 Subject: [PATCH 217/454] Update reset-metadata.py --- tubesync/sync/management/commands/reset-metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/reset-metadata.py b/tubesync/sync/management/commands/reset-metadata.py index be344e1d..f11748ab 100644 --- a/tubesync/sync/management/commands/reset-metadata.py +++ b/tubesync/sync/management/commands/reset-metadata.py @@ -1,5 +1,6 @@ from django.core.management.base import BaseCommand -from sync.models import Media +from common.utils import django_queryset_generator as qs_gen +from sync.models import Media, Metadata from common.logger import log @@ -12,8 +13,8 @@ class Command(BaseCommand): def handle(self, *args, **options): log.info('Resettings all media metadata...') # Delete all metadata - Media.objects.update(metadata=None) + Metadata.objects.all().delete() # Trigger the save signal on each media item - for item in Media.objects.all(): - item.save() + for media in qs_gen(Media.objects.filter(metadata__isnull=False)): + media.metadata_clear(save=True) log.info('Done') From 6caf26ed5bfc9939db6b8e60fe05d13f85f7a1b1 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 26 Apr 2025 22:02:48 -0400 Subject: [PATCH 218/454] Drop an unnecessary 's' --- tubesync/sync/management/commands/reset-metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/management/commands/reset-metadata.py b/tubesync/sync/management/commands/reset-metadata.py index f11748ab..aa003df7 100644 --- a/tubesync/sync/management/commands/reset-metadata.py +++ b/tubesync/sync/management/commands/reset-metadata.py @@ -11,7 +11,7 @@ class Command(BaseCommand): help = 'Resets all media item metadata' def handle(self, *args, **options): - log.info('Resettings all media metadata...') + log.info('Resetting all media metadata...') # Delete all metadata Metadata.objects.all().delete() # Trigger the save signal on each media item From ccf0e98798a9977adc24464576ea05088048316e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 02:20:33 -0400 Subject: [PATCH 219/454] Use the actual model for data migration --- tubesync/sync/migrations/0032_metadata_transfer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0032_metadata_transfer.py b/tubesync/sync/migrations/0032_metadata_transfer.py index ce4b8344..f9e3d53f 100644 --- a/tubesync/sync/migrations/0032_metadata_transfer.py +++ b/tubesync/sync/migrations/0032_metadata_transfer.py @@ -2,16 +2,17 @@ from django.db import migrations from common.utils import django_queryset_generator as qs_gen +from sync.models import Media def use_tables(apps, schema_editor): - Media = apps.get_model('sync', 'Media') + #Media = apps.get_model('sync', 'Media') qs = Media.objects.filter(metadata__isnull=False) for media in qs_gen(qs): media.save_to_metadata('migrated', True) def restore_metadata_column(apps, schema_editor): - Media = apps.get_model('sync', 'Media') + #Media = apps.get_model('sync', 'Media') qs = Media.objects.filter(metadata__isnull=False) for media in qs_gen(qs): metadata = media.loaded_metadata From 832799b51b6123cfc4092248ab73a3251d11005f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 02:31:08 -0400 Subject: [PATCH 220/454] Fix variable scoping --- tubesync/sync/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index f36695c5..b6695ea4 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1851,12 +1851,14 @@ class Metadata(models.Model): @atomic(durable=False) def ingest_formats(self, formats=list(), /): + number = 0 for number, format in enumerate(formats, start=1): mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) mdf.value = format mdf.save() - # delete any numbers we did not overwrite or create - self.format.filter(site=self.site, key=self.key, number__gt=number).delete() + if number > 0: + # delete any numbers we did not overwrite or create + self.format.filter(site=self.site, key=self.key, number__gt=number).delete() @property def with_formats(self): From cdbdc755cb2d6822031094f6622bd4f5e9eb45e8 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 03:05:34 -0400 Subject: [PATCH 221/454] Fix order of operations We must look for the value before we set it. --- tubesync/sync/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index b6695ea4..e9f03b8d 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1144,13 +1144,13 @@ class Media(models.Model): def save_to_metadata(self, key, value, /): data = self.loaded_metadata - data[key] = value - self.ingest_metadata(data) using_new_metadata = self.get_metadata_first_value( ('migrated', '_using_table',), False, arg_dict=data, ) + data[key] = value + self.ingest_metadata(data) if not using_new_metadata: epoch = self.get_metadata_first_value('epoch', arg_dict=data) migrated = dict(migrated=True, epoch=epoch) From 91cb0ce83ba1b6eca91f13379a6530066d6b245e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 03:30:39 -0400 Subject: [PATCH 222/454] Ignore `KeyError` when deleting keys --- tubesync/sync/migrations/0032_metadata_transfer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0032_metadata_transfer.py b/tubesync/sync/migrations/0032_metadata_transfer.py index f9e3d53f..77a37d0e 100644 --- a/tubesync/sync/migrations/0032_metadata_transfer.py +++ b/tubesync/sync/migrations/0032_metadata_transfer.py @@ -16,8 +16,14 @@ def restore_metadata_column(apps, schema_editor): qs = Media.objects.filter(metadata__isnull=False) for media in qs_gen(qs): metadata = media.loaded_metadata - del metadata['migrated'] - del metadata['_using_table'] + try: + del metadata['migrated'] + except KeyError: + pass + try: + del metadata['_using_table'] + except KeyError: + pass media.metadata = media.metadata_dumps(arg_dict=metadata) media.save() From 6d19844af7d2d45213073c90be24febd378ce52d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 03:49:34 -0400 Subject: [PATCH 223/454] `dict.pop` is the nicer way to remove keys --- tubesync/sync/migrations/0032_metadata_transfer.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/migrations/0032_metadata_transfer.py b/tubesync/sync/migrations/0032_metadata_transfer.py index 77a37d0e..1aca0442 100644 --- a/tubesync/sync/migrations/0032_metadata_transfer.py +++ b/tubesync/sync/migrations/0032_metadata_transfer.py @@ -16,14 +16,8 @@ def restore_metadata_column(apps, schema_editor): qs = Media.objects.filter(metadata__isnull=False) for media in qs_gen(qs): metadata = media.loaded_metadata - try: - del metadata['migrated'] - except KeyError: - pass - try: - del metadata['_using_table'] - except KeyError: - pass + for key in {'migrated', '_using_table'}: + metadata.pop(key, None) media.metadata = media.metadata_dumps(arg_dict=metadata) media.save() From b9ee06eeb55d9a22fa31249bd00c6b7529669fc6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 12:58:34 -0400 Subject: [PATCH 224/454] Remove the side-effect from `Media.metadata_loads` --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e9f03b8d..d621f445 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1121,7 +1121,7 @@ class Media(models.Model): def metadata_loads(self, arg_str='{}'): data = json.loads(arg_str) or self.loaded_metadata - return self.ingest_metadata(data) + return data @atomic(durable=False) From 050a60c0c67e632d094a04c7b77fa80873904d0c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 14:57:01 -0400 Subject: [PATCH 225/454] Better result when `Media.new_metadata` does not exist --- tubesync/sync/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e9f03b8d..cecd1f15 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1115,7 +1115,12 @@ class Media(models.Model): def metadata_dumps(self, arg_dict=dict()): from common.utils import json_serial - data = arg_dict or self.new_metadata.with_formats + fallback = dict() + try: + fallback.update(self.new_metadata.with_formats) + except ObjectDoesNotExist: + pass + data = arg_dict or fallback return json.dumps(data, separators=(',', ':'), default=json_serial) From 127fc990eb0c6ff4095b381d6fd31a00caf2c127 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 27 Apr 2025 16:23:51 -0400 Subject: [PATCH 226/454] Use the length of the new metadata --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 50f8ff16..efd03152 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -514,7 +514,7 @@ def download_media_metadata(media_id): # Don't filter media here, the post_save signal will handle that save_model(media) - log.info(f'Saved {len(media.metadata)} bytes of metadata for: ' + log.info(f'Saved {len(media.metadata_dumps())} bytes of metadata for: ' f'{source} / {media}: {media_id}') From e2bf2271873d31f5642850c6cf4d67b8d54336e8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 28 Apr 2025 14:20:34 -0400 Subject: [PATCH 227/454] Add redownload thumb link --- tubesync/sync/templates/sync/media-item.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html index af916caa..b70f78c2 100644 --- a/tubesync/sync/templates/sync/media-item.html +++ b/tubesync/sync/templates/sync/media-item.html @@ -39,6 +39,9 @@
+
+ +
From ea545d49a6541f64fc12e6da6c8dcaf58e4ca431 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 28 Apr 2025 14:39:59 -0400 Subject: [PATCH 228/454] Add `redownload-thumb` URL --- tubesync/sync/urls.py | 190 +++++++++++++++++++++++++++--------------- 1 file changed, 121 insertions(+), 69 deletions(-) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 116beca5..9cec74ee 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -14,104 +14,156 @@ urlpatterns = [ # Dashboard URLs - path('', - DashboardView.as_view(), - name='dashboard'), + path( + '', + DashboardView.as_view(), + name='dashboard', + ), # Source URLs - path('sources', - SourcesView.as_view(), - name='sources'), + path( + 'sources', + SourcesView.as_view(), + name='sources', + ), - path('source-validate/', - ValidateSourceView.as_view(), - name='validate-source'), + path( + 'source-validate/', + ValidateSourceView.as_view(), + name='validate-source', + ), - path('source-sync-now/', - SourcesView.as_view(), - name='source-sync-now'), + path( + 'source-sync-now/', + SourcesView.as_view(), + name='source-sync-now', + ), - path('source-add', - AddSourceView.as_view(), - name='add-source'), + path( + 'source-add', + AddSourceView.as_view(), + name='add-source', + ), - path('source/', - SourceView.as_view(), - name='source'), + path( + 'source/', + SourceView.as_view(), + name='source', + ), - path('source-update/', - UpdateSourceView.as_view(), - name='update-source'), + path( + 'source-update/', + UpdateSourceView.as_view(), + name='update-source', + ), - path('source-delete/', - DeleteSourceView.as_view(), - name='delete-source'), + path( + 'source-delete/', + DeleteSourceView.as_view(), + name='delete-source', + ), # Media URLs - path('media', - MediaView.as_view(), - name='media'), + path( + 'media', + MediaView.as_view(), + name='media', + ), - path('media-thumb/', - MediaThumbView.as_view(), - name='media-thumb'), + path( + 'media-thumb/', + MediaThumbView.as_view(), + name='media-thumb', + ), - path('media/', - MediaItemView.as_view(), - name='media-item'), + path( + 'media/', + MediaItemView.as_view(), + name='media-item', + ), - path('media-redownload/', - MediaRedownloadView.as_view(), - name='redownload-media'), + path( + 'media-redownload/', + MediaRedownloadView.as_view(), + name='redownload-media', + ), - path('media-skip/', - MediaSkipView.as_view(), - name='skip-media'), + path( + 'media-thumb-redownload/', + MediaItemView.as_view(), + name='redownload-thumb', + ), - path('media-enable/', - MediaEnableView.as_view(), - name='enable-media'), + path( + 'media-skip/', + MediaSkipView.as_view(), + name='skip-media', + ), - path('media-content/', - MediaContent.as_view(), - name='media-content'), + path( + 'media-enable/', + MediaEnableView.as_view(), + name='enable-media', + ), + + path( + 'media-content/', + MediaContent.as_view(), + name='media-content', + ), # Task URLs - path('tasks', - TasksView.as_view(), - name='tasks'), + path( + 'tasks', + TasksView.as_view(), + name='tasks', + ), - path('tasks-completed', - CompletedTasksView.as_view(), - name='tasks-completed'), + path( + 'tasks-completed', + CompletedTasksView.as_view(), + name='tasks-completed', + ), - path('tasks-reset', - ResetTasks.as_view(), - name='reset-tasks'), + path( + 'tasks-reset', + ResetTasks.as_view(), + name='reset-tasks', + ), # Media Server URLs - path('mediaservers', - MediaServersView.as_view(), - name='mediaservers'), + path( + 'mediaservers', + MediaServersView.as_view(), + name='mediaservers', + ), - path('mediaserver-add/', - AddMediaServerView.as_view(), - name='add-mediaserver'), + path( + 'mediaserver-add/', + AddMediaServerView.as_view(), + name='add-mediaserver', + ), - path('mediaserver/', - MediaServerView.as_view(), - name='mediaserver'), + path( + 'mediaserver/', + MediaServerView.as_view(), + name='mediaserver', + ), - path('mediaserver-delete/', - DeleteMediaServerView.as_view(), - name='delete-mediaserver'), + path( + 'mediaserver-delete/', + DeleteMediaServerView.as_view(), + name='delete-mediaserver', + ), - path('mediaserver-update/', - UpdateMediaServerView.as_view(), - name='update-mediaserver'), + path( + 'mediaserver-update/', + UpdateMediaServerView.as_view(), + name='update-mediaserver', + ), ] From bd0ad3682a61a709766f0d5cbe266a08c5cb0c2e Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 28 Apr 2025 15:03:13 -0400 Subject: [PATCH 229/454] Handle thumbnail redownload requests --- tubesync/sync/views.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index e7932a86..720b377a 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -546,6 +546,7 @@ class MediaItemView(DetailView): template_name = 'sync/media-item.html' model = Media messages = { + 'thumbnail': _('Thumbnail has been scheduled to redownload'), 'redownloading': _('Media file has been deleted and scheduled to redownload'), 'skipped': _('Media file has been deleted and marked to never download'), 'enabled': _('Media has been re-enabled and will be downloaded'), @@ -581,6 +582,24 @@ class MediaItemView(DetailView): data['media_file_path'] = pathlib.Path(self.object.media_file.path) if self.object.media_file else None return data + def get(self, *args, **kwargs): + if args[0].path.startswith("/media-thumb-redownload/"): + media = Media.objects.get(pk=kwargs["pk"]) + if media is None: + return HttpResponseNotFound() + + verbose_name = _('Redownload thumbnail for "{}": {}') + download_media_thumbnail( + str(media.pk), + media.thumbnail, + verbose_name=verbose_name.format(media.key, media.name), + ) + url = reverse_lazy('sync:media-item') + url = append_uri_params(url, {'message': 'thumbnail'}) + return HttpResponseRedirect(url) + else: + return super().get(self, *args, **kwargs) + class MediaRedownloadView(FormView, SingleObjectMixin): ''' From 0875d09ae83a3e1e3dd782d7178f0e8214a827ed Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 10:18:37 -0400 Subject: [PATCH 230/454] Fix `media-thumb-redownload` errors --- tubesync/sync/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 720b377a..a23597ce 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -30,6 +30,7 @@ from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task, + download_media_thumbnail, check_source_directory_exists, migrate_queues) from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule, YouTube_SourceType, youtube_long_source_types, @@ -594,7 +595,7 @@ class MediaItemView(DetailView): media.thumbnail, verbose_name=verbose_name.format(media.key, media.name), ) - url = reverse_lazy('sync:media-item') + url = reverse_lazy('sync:media-item', kwargs={'pk': media.pk}) url = append_uri_params(url, {'message': 'thumbnail'}) return HttpResponseRedirect(url) else: From 84b1a48f307c95dff3915f43805e746832ac1fcd Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 13:22:09 -0400 Subject: [PATCH 231/454] Overwrite thumbnail files --- tubesync/sync/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a9965a70..a68e1c4b 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -572,8 +572,11 @@ class Source(models.Model): return entries def get_media_thumb_path(instance, filename): - fileid = str(instance.uuid) - filename = f'{fileid.lower()}.jpg' + # we don't want to use alternate names for thumb files + if instance.thumb: + instance.thumb.delete(save=False) + fileid = str(instance.uuid).lower() + filename = f'{fileid}.jpg' prefix = fileid[:2] return Path('thumbs') / prefix / filename From f724c5765ff595993ea86017f2268338dad8c0c0 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 14:52:09 -0400 Subject: [PATCH 232/454] Do not cache the blank thumbnail for so long --- tubesync/sync/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index a23597ce..73fb3ddd 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -523,6 +523,9 @@ class MediaThumbView(DetailView): def get(self, request, *args, **kwargs): media = self.get_object() + # Thumbnail media is never updated so we can ask the browser to cache it + # for ages, 604800 = 7 days + max_age = 604800 if media.thumb_file_exists: thumb_path = pathlib.Path(media.thumb.path) thumb = thumb_path.read_bytes() @@ -532,10 +535,10 @@ class MediaThumbView(DetailView): thumb = b64decode('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAA' 'AAAABAAEAAAICTAEAOw==') content_type = 'image/gif' + max_age = 600 response = HttpResponse(thumb, content_type=content_type) - # Thumbnail media is never updated so we can ask the browser to cache it - # for ages, 604800 = 7 days - response['Cache-Control'] = 'public, max-age=604800' + + response['Cache-Control'] = f'public, max-age={max_age}' return response From f9bb33c3abddc671517f7e45ede5e2209c394fd3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 29 Apr 2025 17:43:35 -0400 Subject: [PATCH 233/454] Save thumbnail when refreshing formats --- tubesync/sync/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a68e1c4b..eec9d7ca 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -25,7 +25,8 @@ from .youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) from .utils import (seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote) + write_text_file, mkdir_p, directory_and_stem, glob_quote, + multi_key_sort) from .matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) from .fields import CommaSepChoiceField @@ -1268,6 +1269,27 @@ class Media(models.Model): if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False): response = filter_response(metadata, True) + # save the new list of thumbnails + thumbnails = self.get_metadata_first_value( + 'thumbnails', + self.get_metadata_first_value('thumbnails', []), + arg_dict=response, + ) + field = self.get_metadata_field('thumbnails') + self.save_to_metadata(field, thumbnails) + + # select and save our best thumbnail url + try: + thumbnail = [ thumb.get('url') for thumb in multi_key_sort( + thumbnails, + [('preference', True,)], + ) if thumb.get('url', '').endswith('.jpg') ][0] + except IndexError: + pass + else: + field = self.get_metadata_field('thumbnail') + self.save_to_metadata(field, thumbnail) + field = self.get_metadata_field('formats') self.save_to_metadata(field, response.get(field, [])) self.save_to_metadata(refreshed_key, response.get('epoch', formats_seconds)) From 6fe76eb34aa5ae294caf64b1fe95f6a519fb6136 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 15:54:21 -0400 Subject: [PATCH 234/454] Adjust metadata & media download delays --- tubesync/tubesync/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index dac5896f..9960f60e 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -135,7 +135,7 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) MAX_ATTEMPTS = 15 # Number of times tasks will be retried -MAX_RUN_TIME = 1*(24*60*60) # Maximum amount of time in seconds a task can run +MAX_RUN_TIME = 12*(60*60) # Maximum amount of time in seconds a task can run BACKGROUND_TASK_RUN_ASYNC = False # Run tasks async in the background BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once MAX_BACKGROUND_TASK_ASYNC_THREADS = 8 # For sanity reasons @@ -173,6 +173,8 @@ YOUTUBE_DEFAULTS = { 'cachedir': False, # Disable on-disk caching 'addmetadata': True, # Embed metadata during postprocessing where available 'geo_verification_proxy': getenv('geo_verification_proxy').strip() or None, + 'max_sleep_interval': (60)*5, + 'sleep_interval': 0.25, } COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' @@ -210,7 +212,7 @@ except: if MAX_RUN_TIME < 600: MAX_RUN_TIME = 600 -DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 50) +DOWNLOAD_MEDIA_DELAY = 1 + round(MAX_RUN_TIME / 100) if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS From 46dc2c9fee157bacc035c8d7cafd1aae94090f51 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 16:35:38 -0400 Subject: [PATCH 235/454] Include `missing_pot` formats in testing --- tubesync/sync/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index ffcbb074..55046c81 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -198,6 +198,7 @@ def get_media_info(url, /, *, days=None, info_json=None): 'clean_infojson': False, 'daterange': yt_dlp.utils.DateRange(start=start), 'extractor_args': { + 'youtube': {'formats': ['missing_pot']}, 'youtubetab': {'approximate_date': ['true']}, }, 'outtmpl': outtmpl, From f8553c7ffca1f96692ff54cea76e21455dd97199 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 17:04:20 -0400 Subject: [PATCH 236/454] Add Proof-of-Origin Token plugin framework --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 2976db2e..17288683 100644 --- a/Pipfile +++ b/Pipfile @@ -24,3 +24,4 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"} emoji = "*" brotli = "*" html5lib = "*" +yt-dlp-get-pot = "*" From 3f073b3ec336ba07391059420e915460db30a077 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 19:09:55 -0400 Subject: [PATCH 237/454] Include openresty --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c21a18c1..3d1d55ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,28 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -f /var/cache/debconf/*.dat-old +FROM alpine:${ALPINE_VERSION} AS openresty-debian +ARG TARGETARCH +ARG DEBIAN_VERSION +ADD 'https://openresty.org/package/pubkey.gpg' '/downloaded/pubkey.gpg' +RUN set -eu ; \ + decide_arch() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '' ;; \ + (arm64) printf -- 'arm64/' ;; \ + esac ; \ + } ; \ + set -x ; \ + mkdir -v -p '/etc/apt/trusted.gpg.d' && \ + apk --no-cache --no-progress add cmd:gpg2 && \ + gpg2 --dearmor \ + -o '/etc/apt/trusted.gpg.d/openresty.gpg' \ + < '/downloaded/pubkey.gpg' && \ + mkdir -v -p '/etc/apt/sources.list.d' && \ + printf -- >| '/etc/apt/sources.list.d/openresty.list' \ + 'deb http://openresty.org/package/%sdebian %s openresty' \ + "$(decide_arch)" "${DEBIAN_VERSION%-slim}" + FROM alpine:${ALPINE_VERSION} AS ffmpeg-download ARG FFMPEG_DATE ARG FFMPEG_VERSION @@ -257,7 +279,36 @@ RUN set -eu ; \ FROM scratch AS s6-overlay COPY --from=s6-overlay-extracted /s6-overlay-rootfs / -FROM tubesync-base AS tubesync +FROM tubesync-base AS tubesync-openresty + +COPY --from=openresty-debian \ + /etc/apt/trusted.gpg.d/openresty.gpg /etc/apt/trusted.gpg.d/openresty.gpg +COPY --from=openresty-debian \ + /etc/apt/sources.list.d/openresty.list /etc/apt/sources.list.d/openresty.list + +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -x && \ + apt-get update && \ + apt-get -y --no-install-recommends install nginx-light openresty && \ + # Clean up + apt-get -y autopurge && \ + apt-get -y autoclean && \ + rm -v -f /var/cache/debconf/*.dat-old + +FROM tubesync-base AS tubesync-nginx + +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -x && \ + apt-get update && \ + apt-get -y --no-install-recommends install nginx-light && \ + # Clean up + apt-get -y autopurge && \ + apt-get -y autoclean && \ + rm -v -f /var/cache/debconf/*.dat-old + +FROM tubesync-openresty AS tubesync ARG S6_VERSION @@ -282,7 +333,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va libmariadb3 \ libpq5 \ libwebp7 \ - nginx-light \ pipenv \ pkgconf \ python3 \ From a8d8e9d055ef4d2e1e43a7eeae557a0b93df5334 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:23:14 -0400 Subject: [PATCH 238/454] Update run --- config/root/etc/s6-overlay/s6-rc.d/nginx/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index 87769e62..63653343 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -2,4 +2,4 @@ cd / -exec /usr/sbin/nginx +exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr From cfb4b4ca1717517b9a5c1f33a7d25747d9445138 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:32:54 -0400 Subject: [PATCH 239/454] Don't install `nginx` binaries --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3d1d55ee..c3a1c5a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -290,7 +290,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ set -x && \ apt-get update && \ - apt-get -y --no-install-recommends install nginx-light openresty && \ + apt-get -y --no-install-recommends install nginx-common openresty && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ From 7ca79af746dc65c345cbadb8c81bf21e12e247aa Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:36:20 -0400 Subject: [PATCH 240/454] Use the `openresty` binary --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c3a1c5a1..64782388 100644 --- a/Dockerfile +++ b/Dockerfile @@ -456,7 +456,8 @@ RUN set -x && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ # Check nginx configuration copied from config/root/etc - nginx -t && \ + openresty -c /etc/nginx/nginx.conf -e stderr + -t && \ # Append software versions ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${ffmpeg_version}" && \ From f9d1e00356f97fa7f769ee5a7280e41b45e97902 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 21:41:57 -0400 Subject: [PATCH 241/454] fixup: remove the extra newline --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64782388..6d49e051 100644 --- a/Dockerfile +++ b/Dockerfile @@ -456,8 +456,7 @@ RUN set -x && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ # Check nginx configuration copied from config/root/etc - openresty -c /etc/nginx/nginx.conf -e stderr - -t && \ + openresty -c /etc/nginx/nginx.conf -e stderr -t && \ # Append software versions ffmpeg_version=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${ffmpeg_version}" && \ From e58c4bae99efefc6f27b2fa790fa6ad55e7f8edd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 22:13:45 -0400 Subject: [PATCH 242/454] Switch back to the `nginx-light` stage --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6d49e051..eabc4f2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -303,12 +303,14 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va set -x && \ apt-get update && \ apt-get -y --no-install-recommends install nginx-light && \ + # openresty binary should still work + ln -v -s -T ../sbin/nginx /usr/bin/openresty && \ # Clean up apt-get -y autopurge && \ apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-openresty AS tubesync +FROM tubesync-nginx AS tubesync ARG S6_VERSION From 144720f6af4d9ee9f4bf289ce3fa42c750d3419c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 30 Apr 2025 22:47:53 -0400 Subject: [PATCH 243/454] Switch to the `openresty` stage --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eabc4f2e..6ef178c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -310,7 +310,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-nginx AS tubesync +FROM tubesync-openresty AS tubesync ARG S6_VERSION From ca9fbd7168ef0bbf9ffc013a3f5d9c66799f8bc6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 03:25:36 -0400 Subject: [PATCH 244/454] Copy updated thumbnail next to media --- tubesync/sync/tasks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index efd03152..81dd4b5d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -555,6 +555,16 @@ def download_media_thumbnail(media_id, url): ) i = image_file = None log.info(f'Saved thumbnail for: {media} from: {url}') + # After media is downloaded, copy the updated thumbnail. + copy_thumbnail = ( + media.downloaded and + media.source.copy_thumbnails and + media.thumb_file_exists + ) + if copy_thumbnail: + log.info(f'Copying media thumbnail from: {media.thumb.path} ' + f'to: {media.thumbpath}') + copyfile(media.thumb.path, media.thumbpath) return True From ddf90122103df88aba9d7ad1635a1bd6624e68a6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 08:42:56 -0400 Subject: [PATCH 245/454] Handle `AssertionError` for timestamp --- tubesync/sync/tasks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 81dd4b5d..ed628958 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -330,9 +330,14 @@ def index_source_task(source_id): media.duration = float(video.get(fields('duration', media), None) or 0) or None media.title = str(video.get(fields('title', media), ''))[:200] timestamp = video.get(fields('timestamp', media), None) - published_dt = media.metadata_published(timestamp) - if published_dt is not None: - media.published = published_dt + if timestamp is not None: + try: + published_dt = media.metadata_published(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt try: media.save() except IntegrityError as e: From f856caa0245fdef50a2f921280f215bf9b94e67d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:21:21 -0400 Subject: [PATCH 246/454] Remove assertion --- tubesync/sync/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index eec9d7ca..3b1f66a9 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1312,7 +1312,6 @@ class Media(models.Model): return self.get_metadata_first_value(('fulltitle', 'title',), '') def ts_to_dt(self, /, timestamp): - assert timestamp is not None try: timestamp_float = float(timestamp) except Exception as e: From 0b37aa3d9f88a4c99e872eac1643a122fa143e31 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:26:14 -0400 Subject: [PATCH 247/454] Call `Media.ts_to_dt` function instead --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ed628958..79f3b375 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -332,7 +332,7 @@ def index_source_task(source_id): timestamp = video.get(fields('timestamp', media), None) if timestamp is not None: try: - published_dt = media.metadata_published(timestamp) + published_dt = media.ts_to_dt(timestamp) except AssertionError: pass else: From 39be70454509605c8ea65f7cd5d019cdc8b2d6d0 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:44:27 -0400 Subject: [PATCH 248/454] Switch to `Media.ts_to_dt` function --- tubesync/sync/tasks.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 79f3b375..39280059 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -505,9 +505,17 @@ def download_media_metadata(media_id): # Media must have a valid upload date if upload_date: media.published = timezone.make_aware(upload_date) - published = media.metadata_published() - if published: - media.published = published + timestamp = media.get_metadata_first_value( + ('release_timestamp', 'timestamp',), + arg_dict=response, + ) + try: + published_dt = media.ts_to_dt(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt # Store title in DB so it's fast to access if media.metadata_title: From c52fd14cd0c206ac750d03930bc75071d7bfe6bb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:46:48 -0400 Subject: [PATCH 249/454] Remove `Media.metadata_published` function --- tubesync/sync/models.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 3b1f66a9..05eaf854 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1321,13 +1321,6 @@ class Media(models.Model): return self.posix_epoch + timedelta(seconds=timestamp_float) return None - def metadata_published(self, timestamp=None): - if timestamp is None: - timestamp = self.get_metadata_first_value( - ('release_timestamp', 'timestamp',) - ) - return self.ts_to_dt(timestamp) - @property def slugtitle(self): replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and') From 19a2d03c5b7c436c31ff951e96d52deb50573c96 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 09:50:49 -0400 Subject: [PATCH 250/454] Remove an unneeded `if` --- tubesync/sync/tasks.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 39280059..5e727e6d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -330,14 +330,13 @@ def index_source_task(source_id): media.duration = float(video.get(fields('duration', media), None) or 0) or None media.title = str(video.get(fields('title', media), ''))[:200] timestamp = video.get(fields('timestamp', media), None) - if timestamp is not None: - try: - published_dt = media.ts_to_dt(timestamp) - except AssertionError: - pass - else: - if published_dt: - media.published = published_dt + try: + published_dt = media.ts_to_dt(timestamp) + except AssertionError: + pass + else: + if published_dt: + media.published = published_dt try: media.save() except IntegrityError as e: From f850502ad05a12184dd186c491ad3c443f951d94 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 1 May 2025 10:24:44 -0400 Subject: [PATCH 251/454] Narrow the exceptions --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 05eaf854..a5e3d675 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1314,7 +1314,7 @@ class Media(models.Model): def ts_to_dt(self, /, timestamp): try: timestamp_float = float(timestamp) - except Exception as e: + except (TypeError, ValueError,) as e: log.warn(f'Could not compute published from timestamp for: {self.source} / {self} with "{e}"') pass else: From cf8555aab1dc3ea308a91ad466640b84f2366695 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 14:28:20 -0400 Subject: [PATCH 252/454] Show migrations when debugging --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index ff0d4d55..baaf6e0c 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -25,6 +25,13 @@ then chmod -R 0755 /downloads fi +if [ 'True' = "${TUBESYNC_DEBUG:-False}" ] +then + s6-setuidgid app \ + /usr/bin/python3 /app/manage.py \ + showmigrations -v 3 --list +fi + # Run migrations exec s6-setuidgid app \ /usr/bin/python3 /app/manage.py migrate From f6b20c03e34431bf5d4cfbcd694b937b2f711af9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 15:36:00 -0400 Subject: [PATCH 253/454] Add `NoThumbnailException` --- tubesync/common/errors.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index 87d8aa4d..67a00eba 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -1,3 +1,6 @@ +from django.http import Http404 + + class NoMediaException(Exception): ''' Raised when a source returns no media to be indexed. Could be an invalid @@ -22,6 +25,13 @@ class NoMetadataException(Exception): pass +class NoThumbnailException(Http404): + ''' + Raised when a thumbnail was not found at the remote URL. + ''' + pass + + class DownloadFailedException(Exception): ''' Raised when a downloaded media file is expected to be present, but doesn't From 3a0c4c8fb15a4deefac2f16027f851510ea6141e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:12:25 -0400 Subject: [PATCH 254/454] Raise when `status_code` is not 200: OK --- tubesync/sync/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 917a9531..b67fedc8 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -65,6 +65,8 @@ def get_remote_image(url, force_rgb=True): '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36') } r = requests.get(url, headers=headers, stream=True, timeout=60) + if 200 != r.status_code: + r.raise_for_status() r.raw.decode_content = True i = Image.open(r.raw) if force_rgb: From add6454c336be23f869a2617fd02d94d5b20a17c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:20:48 -0400 Subject: [PATCH 255/454] Use `NoThumbnailException` from thumbnail task --- tubesync/sync/tasks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 5e727e6d..3c70c0ef 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -8,6 +8,7 @@ import os import json import math import random +import requests import time import uuid from io import BytesIO @@ -29,7 +30,8 @@ from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, - NoMetadataException, DownloadFailedException, ) + NoMetadataException, NoThumbnailException, + DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, ) from .choices import Val, TaskQueue @@ -548,7 +550,10 @@ def download_media_thumbnail(media_id, url): return width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) - i = get_remote_image(url) + try: + i = get_remote_image(url) + except requests.HTTPError as e: + raise NoThumbnailException(url) from e if (i.width > width) and (i.height > height): log.info(f'Resizing {i.width}x{i.height} thumbnail to ' f'{width}x{height}: {url}') From d6b60f41c9fe266d0d169bf168af695a4254ae75 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 16:52:20 -0400 Subject: [PATCH 256/454] End the task for HTTP 404 response status --- tubesync/sync/tasks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3c70c0ef..744373da 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -551,9 +551,14 @@ def download_media_thumbnail(media_id, url): width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) try: - i = get_remote_image(url) - except requests.HTTPError as e: - raise NoThumbnailException(url) from e + try: + i = get_remote_image(url) + except requests.HTTPError as re: + if 404 != re.response.status_code: + raise + raise NoThumbnailException(re.response.reason) from re + except NoThumbnailException as e: + raise InvalidTaskError(str(e.__cause__)) from e if (i.width > width) and (i.height > height): log.info(f'Resizing {i.width}x{i.height} thumbnail to ' f'{width}x{height}: {url}') From a7fdd02d47a462aac003a5de3ebd331d0d56c20f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 17:09:23 -0400 Subject: [PATCH 257/454] A standard `Exception` is fine --- tubesync/common/errors.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index 67a00eba..9ff44a48 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -1,6 +1,3 @@ -from django.http import Http404 - - class NoMediaException(Exception): ''' Raised when a source returns no media to be indexed. Could be an invalid @@ -25,7 +22,7 @@ class NoMetadataException(Exception): pass -class NoThumbnailException(Http404): +class NoThumbnailException(Exception): ''' Raised when a thumbnail was not found at the remote URL. ''' From 46d2e3c8e70243cef3134845e7857ff627034a5e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 17:14:07 -0400 Subject: [PATCH 258/454] `raise_for_status` checks `status_code` itself --- tubesync/sync/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index b67fedc8..5bc90d25 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -65,8 +65,7 @@ def get_remote_image(url, force_rgb=True): '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36') } r = requests.get(url, headers=headers, stream=True, timeout=60) - if 200 != r.status_code: - r.raise_for_status() + r.raise_for_status() r.raw.decode_content = True i = Image.open(r.raw) if force_rgb: From 71b57dd477896cfc23082d5025de6f5ee50b1fa6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 21:03:09 -0400 Subject: [PATCH 259/454] Schedule thumbnail tasks for new media --- tubesync/sync/tasks.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 744373da..304b38e6 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -354,8 +354,21 @@ def index_source_task(source_id): ) if new_media_instance: log.info(f'Indexed new media: {source} / {media}') + log.info(f'Scheduling tasks to download thumbnail for: {media.key}') + thumbnail_fmt = 'https://i.ytimg.com/vi/{}/{}default.jpg' + vn_fmt = _('Downloading {} thumbnail for: "{}": {}' + for prefix in ('hq', 'sd', 'maxres',): + thumbnail_url = thumbnail_fmt.format( + media.key, + prefix, + ) + download_media_thumbnail( + str(media.pk), + thumbnail_url, + verbose_name=vn_fmt.format(prefix, media.key, media.name), + ) log.info(f'Scheduling task to download metadata for: {media.url}') - verbose_name = _('Downloading metadata for: {}: "{}"') + verbose_name = _('Downloading metadata for: "{}": {}') download_media_metadata( str(media.pk), verbose_name=verbose_name.format(media.key, media.name), From efc59420516d480264ecda37afe155e08f12ba1a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 2 May 2025 21:05:16 -0400 Subject: [PATCH 260/454] fixup: closing parenthesis --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 304b38e6..2e6b1433 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -356,7 +356,7 @@ def index_source_task(source_id): log.info(f'Indexed new media: {source} / {media}') log.info(f'Scheduling tasks to download thumbnail for: {media.key}') thumbnail_fmt = 'https://i.ytimg.com/vi/{}/{}default.jpg' - vn_fmt = _('Downloading {} thumbnail for: "{}": {}' + vn_fmt = _('Downloading {} thumbnail for: "{}": {}') for prefix in ('hq', 'sd', 'maxres',): thumbnail_url = thumbnail_fmt.format( media.key, From d5045d8d03232e726058b500b5a9b03b3198fb7a Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 21:13:16 -0400 Subject: [PATCH 261/454] Created fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 tubesync/sync/management/commands/fix-mariadb.py diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py new file mode 100755 index 00000000..ab9d3624 --- /dev/null +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -0,0 +1,39 @@ +from django import db +from django.utils.translation import gettext_lazy as _ +from django.core.management.base import BaseCommand, CommandError +from common.logger import log + + +class Command(BaseCommand): + + help = _('Fixes MariaDB database issues') + requires_migrations_checks = True + + def add_arguments(self, parser): + parser.add_argument( + '--uuid-columns', + action='store_true', + required=False, + help=_('Switch to the native UUID column type'), + ) + parser.add_argument( + '--delete-table', + action='store', + required=False, + help=_('Table name'), + ) + + def handle(self, *args, **options): + if 'mysql' != db.connection.vendor: + raise CommandError( + _('An invalid database vendor is configured') + f': {db.connection.vendor}' + ) + if not db.connection.mysql_is_mariadb(): + raise CommandError(_('Not conbected to a MariaDB database server.')) + + uuid_columns = options.get('uuid-columns', False) + table_name_str = options.get('delete-table', '') + + # All done + log.info('Done') From c11e78bc4704464e183e3f5fa42f021cf0ccc48e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 21:19:59 -0400 Subject: [PATCH 262/454] Remove executable bit --- tubesync/sync/management/commands/fix-mariadb.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tubesync/sync/management/commands/fix-mariadb.py diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py old mode 100755 new mode 100644 From 0f7643971ff88966cf64fe10665d6225688dfe7b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 22:44:31 -0400 Subject: [PATCH 263/454] Adjust options in fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index ab9d3624..a2e290b2 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,9 +1,16 @@ from django import db +from pprint import pp from django.utils.translation import gettext_lazy as _ from django.core.management.base import BaseCommand, CommandError from common.logger import log +def SQLTable(arg_table): + assert isinstance(arg_table, str), type(arg_table) + assert arg_table.startswith('sync_'), _('Invalid table name') + return str(arg_table) + + class Command(BaseCommand): help = _('Fixes MariaDB database issues') @@ -13,14 +20,15 @@ class Command(BaseCommand): parser.add_argument( '--uuid-columns', action='store_true', - required=False, + default=False, help=_('Switch to the native UUID column type'), ) parser.add_argument( '--delete-table', - action='store', - required=False, - help=_('Table name'), + action='append', + metavar='TABLE', + type=SQLTable, + help=_('SQL table name'), ) def handle(self, *args, **options): @@ -32,8 +40,13 @@ class Command(BaseCommand): if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) - uuid_columns = options.get('uuid-columns', False) - table_name_str = options.get('delete-table', '') + uuid_columns = options.get('uuid-columns') + table_names = options.get('delete-table', list()) + + if options['uuid-columns']: + self.stdout.write('Time to update the columns!') + + pp( table_names ) # All done log.info('Done') From a7fe0d5eeb267f48fefaf0a853c4a3af82615866 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 3 May 2025 22:49:13 -0400 Subject: [PATCH 264/454] Combine the strings --- tubesync/sync/management/commands/fix-mariadb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index a2e290b2..bd3dd2bf 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -33,10 +33,10 @@ class Command(BaseCommand): def handle(self, *args, **options): if 'mysql' != db.connection.vendor: - raise CommandError( - _('An invalid database vendor is configured') + raise CommandError(_( + 'An invalid database vendor is configured' f': {db.connection.vendor}' - ) + )) if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) From 07f689e5d0d75edfa06242a687da6dc195df899a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 00:04:00 -0400 Subject: [PATCH 265/454] Update fix-mariadb.py --- .../sync/management/commands/fix-mariadb.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index bd3dd2bf..800b02b8 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,13 +1,18 @@ from django import db from pprint import pp -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand, CommandError from common.logger import log +def _(arg_str): + return str(gettext_lazy(arg_str)) + def SQLTable(arg_table): - assert isinstance(arg_table, str), type(arg_table) - assert arg_table.startswith('sync_'), _('Invalid table name') + if not isinstance(arg_table, str): + raise TypeError(type(arg_table)) + if not arg_table.startswith('sync_'): + raise ValueError(_('Invalid table name')) return str(arg_table) @@ -33,17 +38,16 @@ class Command(BaseCommand): def handle(self, *args, **options): if 'mysql' != db.connection.vendor: - raise CommandError(_( - 'An invalid database vendor is configured' - f': {db.connection.vendor}' - )) + raise CommandError( + _('An invalid database vendor is configured') + + f': {db.connection.vendor}' + ) if not db.connection.mysql_is_mariadb(): raise CommandError(_('Not conbected to a MariaDB database server.')) - uuid_columns = options.get('uuid-columns') - table_names = options.get('delete-table', list()) + table_names = options.get('delete_table', list()) - if options['uuid-columns']: + if options['uuid_columns']: self.stdout.write('Time to update the columns!') pp( table_names ) From 919ba49e28fb0c281481b12ff8f5f8d3a2350eaf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 00:21:16 -0400 Subject: [PATCH 266/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 800b02b8..a25f8da7 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -31,6 +31,7 @@ class Command(BaseCommand): parser.add_argument( '--delete-table', action='append', + default=list(), metavar='TABLE', type=SQLTable, help=_('SQL table name'), @@ -42,14 +43,20 @@ class Command(BaseCommand): _('An invalid database vendor is configured') + f': {db.connection.vendor}' ) - if not db.connection.mysql_is_mariadb(): + db_is_mariadb = ( + hasattr(db.connection, 'mysql_is_mariadb') and + db.connection.mysql_is_mariadb() + ) + if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) - table_names = options.get('delete_table', list()) + table_names = options.get('delete_table') + log.info('Start') if options['uuid_columns']: self.stdout.write('Time to update the columns!') + self.stdout.write('Tables to drop:') pp( table_names ) # All done From 960c07a84665e1f532c4db21bc06708fa937e204 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 01:20:39 -0400 Subject: [PATCH 267/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index a25f8da7..826b5533 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -11,7 +11,12 @@ def _(arg_str): def SQLTable(arg_table): if not isinstance(arg_table, str): raise TypeError(type(arg_table)) - if not arg_table.startswith('sync_'): + tables = db.connection.introspection.table_names(include_views=False) + valid_table_name = ( + arg_table.startswith('sync_') and + arg_table in tables + ) + if not valid_table_name: raise ValueError(_('Invalid table name')) return str(arg_table) @@ -43,20 +48,27 @@ class Command(BaseCommand): _('An invalid database vendor is configured') + f': {db.connection.vendor}' ) + db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and + db.connection.is_usable() and db.connection.mysql_is_mariadb() ) if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) + display_name = db.connection.display_name table_names = options.get('delete_table') log.info('Start') if options['uuid_columns']: + if 'uuid' != db.connection.data_types.get('UUIDField', ''): + raise CommandError(_( + f'The {display_name} database server does not support UUID columns.' + )) self.stdout.write('Time to update the columns!') - self.stdout.write('Tables to drop:') + self.stdout.write('Tables to delete:') pp( table_names ) # All done From e4f3428aa8c2ccaa46f1bf21251e432b4a38177d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 01:49:18 -0400 Subject: [PATCH 268/454] Allow deleting only a few tables - It must be one of the metadata related tables - The table must be present in the database --- .../sync/management/commands/fix-mariadb.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 826b5533..152111f8 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -5,16 +5,25 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db_tables = db.connection.introspection.table_names +new_tables = { + 'sync_media_metadata_format', + 'sync_media_metadata', + 'sync_metadataformat', + 'sync_metadata', +} + def _(arg_str): return str(gettext_lazy(arg_str)) def SQLTable(arg_table): - if not isinstance(arg_table, str): - raise TypeError(type(arg_table)) - tables = db.connection.introspection.table_names(include_views=False) + assert isinstance(arg_table, str), type(arg_table) + needle = arg_table + if needle.startswith('new__'): + needle = arg_table[len('new__'):] valid_table_name = ( - arg_table.startswith('sync_') and - arg_table in tables + needle in new_tables and + arg_table in db_tables(include_views=False) ) if not valid_table_name: raise ValueError(_('Invalid table name')) From 5235c53dadce7d5e72f142c41b5c3bb1cf2fee3f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 03:47:18 -0400 Subject: [PATCH 269/454] Check the current column type --- .../sync/management/commands/fix-mariadb.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 152111f8..cb6b5c1d 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -5,7 +5,9 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db_columns = db.connection.introspection.get_table_description db_tables = db.connection.introspection.table_names +db_quote_name = db.connection.ops.quote_name new_tables = { 'sync_media_metadata_format', 'sync_media_metadata', @@ -51,6 +53,20 @@ class Command(BaseCommand): help=_('SQL table name'), ) + def _get_fields(self, table_str, /): + columns = list() + with db.connection.cursor() as cursor: + columns.extend(db_columns(cursor, table_str)) + return columns + + def _using_char_for_uuid(self, table_str, /): + fields = self._get_fields(table_str) + return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] + + def _uuid_column_type(self, table_str, /): + fields = self._get_fields(table_str) + return [ f.type_code for f in fields if 'uuid' == f.name ][0] + def handle(self, *args, **options): if 'mysql' != db.connection.vendor: raise CommandError( @@ -75,7 +91,22 @@ class Command(BaseCommand): raise CommandError(_( f'The {display_name} database server does not support UUID columns.' )) - self.stdout.write('Time to update the columns!') + both_tables = ( + self._using_char_for_uuid('sync_source') and + self._using_char_for_uuid('sync_media') + ) + if not both_tables: + if 'uuid' == self._uuid_column_type('sync_source').lower(): + log.notice('The source table is already using a native UUID column.') + elif 'uuid' == self._uuid_column_type('sync_media').lower(): + log.notice('The media table is already using a native UUID column.') + else: + raise CommandError(_( + 'The database is not in an appropriate state to switch to ' + 'native UUID columns. Manual intervention is required.' + )) + else: + self.stdout.write('Time to update the columns!') self.stdout.write('Tables to delete:') pp( table_names ) From ba87e39e3d710b2820257cbb5840eddbcc862f8d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 03:58:56 -0400 Subject: [PATCH 270/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index cb6b5c1d..24eabcee 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -63,9 +63,9 @@ class Command(BaseCommand): fields = self._get_fields(table_str) return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] - def _uuid_column_type(self, table_str, /): + def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.type_code for f in fields if 'uuid' == f.name ][0] + return [ f.type_code for f in fields if column_str.lower() == f.name.lower() ][0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -96,9 +96,11 @@ class Command(BaseCommand): self._using_char_for_uuid('sync_media') ) if not both_tables: - if 'uuid' == self._uuid_column_type('sync_source').lower(): + if 'uuid' == self._column_type('sync_source', 'uuid').lower(): log.notice('The source table is already using a native UUID column.') - elif 'uuid' == self._uuid_column_type('sync_media').lower(): + elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): + log.notice('The media table is already using a native UUID column.') + elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): log.notice('The media table is already using a native UUID column.') else: raise CommandError(_( From 435ebac328d40cb7954a951f454148603016b6bb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 05:19:59 -0400 Subject: [PATCH 271/454] Generate some SQL for native UUID columns --- .../sync/management/commands/fix-mariadb.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 24eabcee..57f67804 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -14,6 +14,7 @@ new_tables = { 'sync_metadataformat', 'sync_metadata', } +sql_statements = db.connection.ops.prepare_sql_script def _(arg_str): return str(gettext_lazy(arg_str)) @@ -110,6 +111,52 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') + schema = db.connection.schema_editor() + media_table_str = db_quote_name('sync_media') + source_table_str = db_quote_name('sync_source') + fk_name_str = db_quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') + source_id_column_str = db_quote_name('source_id') + uuid_column_str = db_quote_name('uuid') + uuid_type_str = 'uuid'.upper() + remove_fk = schema.sql_delete_fk.format(dict( + table=media_table_str, + name=fk_name_str, + )) + add_fk = schema.sql_create_fk.format(dict( + table=media_table_str, + name=fk_name_str, + column=source_id_column_str, + to_table=source_table_str, + to_column=uuid_column_str, + deferrable='', + )) + statement_list = list(( + remove_fk, + schema.sql_alter_column.format(dict( + table=media_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=uuid_column_str, + )), + )), + schema.sql_alter_column.format(dict( + table=media_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=source_id_column_str, + )), + )), + schema.sql_alter_column.format(dict( + table=source_table_str, + changes=schema.sql_alter_column_not_null.format(dict( + type=uuid_type_str, + column=uuid_column_str, + )), + )), + add_fk, + )) + pp( statement_list ) + self.stdout.write('Tables to delete:') pp( table_names ) From 503bb3e5b197eeccaa3f61454472265faff32814 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 05:46:33 -0400 Subject: [PATCH 272/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 57f67804..23919d2e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -62,11 +62,11 @@ class Command(BaseCommand): def _using_char_for_uuid(self, table_str, /): fields = self._get_fields(table_str) - return 'uuid' in [ f.name for f in fields if 'char(32)' == f.type_code ] + return 'uuid' in [ f.name for f in fields if 'varchar' == f.data_type ] def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.type_code for f in fields if column_str.lower() == f.name.lower() ][0] + return [ f.data_type for f in fields if column_str.lower() == f.name.lower() ][0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -78,7 +78,7 @@ class Command(BaseCommand): db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and - db.connection.mysql_is_mariadb() + db.connection.mysql_is_mariadb ) if not db_is_mariadb: raise CommandError(_('Not conbected to a MariaDB database server.')) @@ -98,11 +98,11 @@ class Command(BaseCommand): ) if not both_tables: if 'uuid' == self._column_type('sync_source', 'uuid').lower(): - log.notice('The source table is already using a native UUID column.') + log.info('The source table is already using a native UUID column.') elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): - log.notice('The media table is already using a native UUID column.') + log.info('The media table is already using a native UUID column.') elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): - log.notice('The media table is already using a native UUID column.') + log.info('The media table is already using a native UUID column.') else: raise CommandError(_( 'The database is not in an appropriate state to switch to ' From fc89178f28ea82c39033006ce6d7282c9c71a9e2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 06:24:10 -0400 Subject: [PATCH 273/454] Switch to string interpolation --- .../sync/management/commands/fix-mariadb.py | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 23919d2e..2978bcea 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -62,11 +62,18 @@ class Command(BaseCommand): def _using_char_for_uuid(self, table_str, /): fields = self._get_fields(table_str) - return 'uuid' in [ f.name for f in fields if 'varchar' == f.data_type ] + return 'uuid' in [ + f.name for f in fields if 'varchar' == f.data_type and 32 == f.display_size + ] def _column_type(self, table_str, column_str='uuid', /): fields = self._get_fields(table_str) - return [ f.data_type for f in fields if column_str.lower() == f.name.lower() ][0] + found = [ + f'{f.data_type}({f.display_size})' for f in fields if column_str.lower() == f.name.lower() + ] + if not found: + return str() + return found[0] def handle(self, *args, **options): if 'mysql' != db.connection.vendor: @@ -92,16 +99,17 @@ class Command(BaseCommand): raise CommandError(_( f'The {display_name} database server does not support UUID columns.' )) + uuid_column_type_str = 'uuid(36)' both_tables = ( self._using_char_for_uuid('sync_source') and self._using_char_for_uuid('sync_media') ) if not both_tables: - if 'uuid' == self._column_type('sync_source', 'uuid').lower(): + if uuid_column_type_str == self._column_type('sync_source', 'uuid').lower(): log.info('The source table is already using a native UUID column.') - elif 'uuid' == self._column_type('sync_media', 'uuid').lower(): + elif uuid_column_type_str == self._column_type('sync_media', 'uuid').lower(): log.info('The media table is already using a native UUID column.') - elif 'uuid' == self._column_type('sync_media', 'source_id').lower(): + elif uuid_column_type_str == self._column_type('sync_media', 'source_id').lower(): log.info('The media table is already using a native UUID column.') else: raise CommandError(_( @@ -118,41 +126,41 @@ class Command(BaseCommand): source_id_column_str = db_quote_name('source_id') uuid_column_str = db_quote_name('uuid') uuid_type_str = 'uuid'.upper() - remove_fk = schema.sql_delete_fk.format(dict( + remove_fk = schema.sql_delete_fk % dict( table=media_table_str, name=fk_name_str, - )) - add_fk = schema.sql_create_fk.format(dict( + ) + add_fk = schema.sql_create_fk % dict( table=media_table_str, name=fk_name_str, column=source_id_column_str, to_table=source_table_str, to_column=uuid_column_str, deferrable='', - )) + ) statement_list = list(( remove_fk, - schema.sql_alter_column.format(dict( + schema.sql_alter_column % dict( table=media_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, - )), - )), - schema.sql_alter_column.format(dict( + ), + ), + schema.sql_alter_column % dict( table=media_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=source_id_column_str, - )), - )), - schema.sql_alter_column.format(dict( + ), + ), + schema.sql_alter_column % dict( table=source_table_str, - changes=schema.sql_alter_column_not_null.format(dict( + changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, - )), - )), + ), + ), add_fk, )) pp( statement_list ) From b5828801de10f240ccc8f9f7ba34ae76db97a53b Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 06:34:09 -0400 Subject: [PATCH 274/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 2978bcea..910b2276 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -138,7 +138,7 @@ class Command(BaseCommand): to_column=uuid_column_str, deferrable='', ) - statement_list = list(( + statement_list = [ f'{statement};' for statement in ( remove_fk, schema.sql_alter_column % dict( table=media_table_str, @@ -162,7 +162,7 @@ class Command(BaseCommand): ), ), add_fk, - )) + ) ] pp( statement_list ) self.stdout.write('Tables to delete:') From cc9b5446c1f15da29604630133b812b6f1b8f4fb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 07:24:54 -0400 Subject: [PATCH 275/454] Use the `quote_name` from schema --- tubesync/sync/management/commands/fix-mariadb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 910b2276..f946c22e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -120,11 +120,12 @@ class Command(BaseCommand): self.stdout.write('Time to update the columns!') schema = db.connection.schema_editor() - media_table_str = db_quote_name('sync_media') - source_table_str = db_quote_name('sync_source') - fk_name_str = db_quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') - source_id_column_str = db_quote_name('source_id') - uuid_column_str = db_quote_name('uuid') + quote_name = schema.quote_name + media_table_str = quote_name('sync_media') + source_table_str = quote_name('sync_source') + fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') + source_id_column_str = quote_name('source_id') + uuid_column_str = quote_name('uuid') uuid_type_str = 'uuid'.upper() remove_fk = schema.sql_delete_fk % dict( table=media_table_str, From 2a98707254c83908931da73b1be4443388cc701e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 07:46:36 -0400 Subject: [PATCH 276/454] Collect SQL with schema editor --- .../sync/management/commands/fix-mariadb.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index f946c22e..9c7ceb87 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -119,7 +119,7 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') - schema = db.connection.schema_editor() + schema = db.connection.schema_editor(collect_sql=True) quote_name = schema.quote_name media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') @@ -139,8 +139,9 @@ class Command(BaseCommand): to_column=uuid_column_str, deferrable='', ) - statement_list = [ f'{statement};' for statement in ( - remove_fk, + + schema.execute(remove_fk, None) + schema.execute( schema.sql_alter_column % dict( table=media_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -148,6 +149,9 @@ class Command(BaseCommand): column=uuid_column_str, ), ), + None, + ) + schema.execute( schema.sql_alter_column % dict( table=media_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -155,6 +159,9 @@ class Command(BaseCommand): column=source_id_column_str, ), ), + None, + ) + schema.execute( schema.sql_alter_column % dict( table=source_table_str, changes=schema.sql_alter_column_not_null % dict( @@ -162,9 +169,11 @@ class Command(BaseCommand): column=uuid_column_str, ), ), - add_fk, - ) ] - pp( statement_list ) + None, + ) + schema.execute(add_fk, None) + + pp( schema.collected_sql ) self.stdout.write('Tables to delete:') pp( table_names ) From 36681aadc0ac66943147bb1e17d38a8b2a54b3ad Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 09:35:04 -0400 Subject: [PATCH 277/454] Add `check_migration_status` function --- .../sync/management/commands/fix-mariadb.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 9c7ceb87..3bbce4c5 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,6 +1,8 @@ from django import db +from io import BytesIO, TextIOWrapper from pprint import pp from django.utils.translation import gettext_lazy +from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError from common.logger import log @@ -32,6 +34,32 @@ def SQLTable(arg_table): raise ValueError(_('Invalid table name')) return str(arg_table) +def _mk_wrapper(): + return TextIOWrapper( + BytesIO(), + line_buffering=True, + write_through=True, + ) + +def check_migration_status(migration_str, /): + needle = 'No planned migration operations.' + wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() + call_command( + 'migrate', '-v', '3', '--plan', 'sync', + migration_str, + stderr=wrap_stderr, + stdout=wrap_stdout, + ) + wrap_stderr.seek(0, 0) + stderr_lines = wrap_stderr.readlines() + wrap_stdout.seek(0, 0) + stdout_lines = wrap_stdout.readlines() + return ( + bool([ line.decode() for line in stdout_lines if needle.encode() in line ]), + stderr_lines, + stdout_lines, + ) + class Command(BaseCommand): From e6e4c3300af8cf8b504057c7e6de1c07581fea63 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 09:41:21 -0400 Subject: [PATCH 278/454] Update fix-mariadb.py --- tubesync/sync/management/commands/fix-mariadb.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 3bbce4c5..39735d6e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -203,8 +203,16 @@ class Command(BaseCommand): pp( schema.collected_sql ) - self.stdout.write('Tables to delete:') - pp( table_names ) + if table_names: + pp( check_migration_status( + '0030_alter_source_source_vcodec', + )) + pp( check_migration_status( + '0031_squashed_metadata_metadataformat', + )) + + self.stdout.write('Tables to delete:') + pp( table_names ) # All done log.info('Done') From 5e2200382f35b30646374fc2539f3d11e4e53977 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 10:10:34 -0400 Subject: [PATCH 279/454] Check for completed data migration --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 39735d6e..387e913a 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -55,7 +55,7 @@ def check_migration_status(migration_str, /): wrap_stdout.seek(0, 0) stdout_lines = wrap_stdout.readlines() return ( - bool([ line.decode() for line in stdout_lines if needle.encode() in line ]), + bool([ line.decode() for line in stdout_lines if needle in line ]), stderr_lines, stdout_lines, ) @@ -64,7 +64,7 @@ def check_migration_status(migration_str, /): class Command(BaseCommand): help = _('Fixes MariaDB database issues') - requires_migrations_checks = True + requires_migrations_checks = False def add_arguments(self, parser): parser.add_argument( @@ -204,12 +204,18 @@ class Command(BaseCommand): pp( schema.collected_sql ) if table_names: + pp( check_migration_status( '0029', ) ) pp( check_migration_status( '0030_alter_source_source_vcodec', )) pp( check_migration_status( '0031_squashed_metadata_metadataformat', )) + pp( check_migration_status( '0032_metadata_transfer', )) + if check_migration_status('0032_metadata_transfer')[0]: + raise CommandError(_( + 'Deleting tables that are in use is not safe!' + )) self.stdout.write('Tables to delete:') pp( table_names ) From 5e8cc639f76b75f2f8cfb94efc5c37beda57d94c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 11:03:26 -0400 Subject: [PATCH 280/454] Generate SQL for deleting tables --- .../sync/management/commands/fix-mariadb.py | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 387e913a..1e5a9ff1 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -41,8 +41,9 @@ def _mk_wrapper(): write_through=True, ) -def check_migration_status(migration_str, /): - needle = 'No planned migration operations.' +def check_migration_status(migration_str, /, *, needle=None): + if needle is None: + needle = 'No planned migration operations.' wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() call_command( 'migrate', '-v', '3', '--plan', 'sync', @@ -55,7 +56,7 @@ def check_migration_status(migration_str, /): wrap_stdout.seek(0, 0) stdout_lines = wrap_stdout.readlines() return ( - bool([ line.decode() for line in stdout_lines if needle in line ]), + bool([ line for line in stdout_lines if needle in line ]), stderr_lines, stdout_lines, ) @@ -120,6 +121,8 @@ class Command(BaseCommand): display_name = db.connection.display_name table_names = options.get('delete_table') + schema = db.connection.schema_editor(collect_sql=True) + quote_name = schema.quote_name log.info('Start') if options['uuid_columns']: @@ -147,8 +150,6 @@ class Command(BaseCommand): else: self.stdout.write('Time to update the columns!') - schema = db.connection.schema_editor(collect_sql=True) - quote_name = schema.quote_name media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') @@ -204,21 +205,34 @@ class Command(BaseCommand): pp( schema.collected_sql ) if table_names: - pp( check_migration_status( '0029', ) ) - pp( check_migration_status( + # Check that the migration is at an appropriate step + at_30, err_30, out_30 = check_migration_status( '0030_alter_source_source_vcodec' ) + at_31, err_31, out_31 = check_migration_status( '0031_metadata_metadataformat' ) + at_31s, err_31s, out_31s = check_migration_status( '0031_squashed_metadata_metadataformat' ) + after_31, err_31a, out_31a = check_migration_status( '0030_alter_source_source_vcodec', - )) - pp( check_migration_status( - '0031_squashed_metadata_metadataformat', - )) - pp( check_migration_status( '0032_metadata_transfer', )) - if check_migration_status('0032_metadata_transfer')[0]: + needle='Undo Rename table for metadata to sync_media_metadata', + ) + + should_delete = ( + not (at_31s or after_31) and + (at_30 or at_31) + ) + if not should_delete: raise CommandError(_( - 'Deleting tables that are in use is not safe!' + 'Deleting metadata tables that are in use is not safe!' )) self.stdout.write('Tables to delete:') pp( table_names ) + for table in table_names: + schema.execute( + schema.sql_delete_table % dict( + table=quote_name(table), + ), + None, + ) + pp( schema.collected_sql ) # All done log.info('Done') From da06a1ffa1f9468a717bb719e173ffff09fe025d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:00:28 -0400 Subject: [PATCH 281/454] Add `--dry-run` and apply SQL when it is not used --- .../sync/management/commands/fix-mariadb.py | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 1e5a9ff1..7f19ff97 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -7,7 +7,6 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log -db_columns = db.connection.introspection.get_table_description db_tables = db.connection.introspection.table_names db_quote_name = db.connection.ops.quote_name new_tables = { @@ -45,12 +44,15 @@ def check_migration_status(migration_str, /, *, needle=None): if needle is None: needle = 'No planned migration operations.' wrap_stderr, wrap_stdout = _mk_wrapper(), _mk_wrapper() - call_command( - 'migrate', '-v', '3', '--plan', 'sync', - migration_str, - stderr=wrap_stderr, - stdout=wrap_stdout, - ) + try: + call_command( + 'migrate', '-v', '3', '--plan', 'sync', + migration_str, + stderr=wrap_stderr, + stdout=wrap_stdout, + ) + except db.migrations.exceptions.NodeNotFoundError: + return (False, None, None,) wrap_stderr.seek(0, 0) stderr_lines = wrap_stderr.readlines() wrap_stdout.seek(0, 0) @@ -61,13 +63,27 @@ def check_migration_status(migration_str, /, *, needle=None): stdout_lines, ) +def db_columns(table_str, /): + columns = list() + db_gtd = db.connection.introspection.get_table_description + with db.connection.cursor() as cursor: + columns.extend(db_gtd(cursor, table_str)) + return columns + class Command(BaseCommand): help = _('Fixes MariaDB database issues') + output_transaction = True requires_migrations_checks = False def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + default=False, + help=_('Only show the SQL; do not apply it to the database'), + ) parser.add_argument( '--uuid-columns', action='store_true', @@ -80,25 +96,21 @@ class Command(BaseCommand): default=list(), metavar='TABLE', type=SQLTable, - help=_('SQL table name'), + help=_('SQL table name to be deleted'), ) - def _get_fields(self, table_str, /): - columns = list() - with db.connection.cursor() as cursor: - columns.extend(db_columns(cursor, table_str)) - return columns - - def _using_char_for_uuid(self, table_str, /): - fields = self._get_fields(table_str) - return 'uuid' in [ - f.name for f in fields if 'varchar' == f.data_type and 32 == f.display_size + def _using_char(self, table_str, column_str='uuid', /): + cols = db_columns(table_str) + char_sizes = { 32, 36, } + char_types = { 'char', 'varchar', } + return column_str in [ + c.name for c in cols if c.data_type in char_types and c.display_size in char_sizes ] def _column_type(self, table_str, column_str='uuid', /): - fields = self._get_fields(table_str) + cols = db_columns(table_str) found = [ - f'{f.data_type}({f.display_size})' for f in fields if column_str.lower() == f.name.lower() + f'{c.data_type}({c.display_size})' for c in cols if column_str.lower() == c.name.lower() ] if not found: return str() @@ -125,6 +137,8 @@ class Command(BaseCommand): quote_name = schema.quote_name log.info('Start') + + if options['uuid_columns']: if 'uuid' != db.connection.data_types.get('UUIDField', ''): raise CommandError(_( @@ -132,8 +146,8 @@ class Command(BaseCommand): )) uuid_column_type_str = 'uuid(36)' both_tables = ( - self._using_char_for_uuid('sync_source') and - self._using_char_for_uuid('sync_media') + self._using_char('sync_source', 'uuid') and + self._using_char('sync_media', 'uuid') ) if not both_tables: if uuid_column_type_str == self._column_type('sync_source', 'uuid').lower(): @@ -148,8 +162,6 @@ class Command(BaseCommand): 'native UUID columns. Manual intervention is required.' )) else: - self.stdout.write('Time to update the columns!') - media_table_str = quote_name('sync_media') source_table_str = quote_name('sync_source') fk_name_str = quote_name('sync_media_source_id_36827e1d_fk_sync_source_uuid') @@ -202,7 +214,6 @@ class Command(BaseCommand): ) schema.execute(add_fk, None) - pp( schema.collected_sql ) if table_names: # Check that the migration is at an appropriate step @@ -223,8 +234,6 @@ class Command(BaseCommand): 'Deleting metadata tables that are in use is not safe!' )) - self.stdout.write('Tables to delete:') - pp( table_names ) for table in table_names: schema.execute( schema.sql_delete_table % dict( @@ -232,7 +241,15 @@ class Command(BaseCommand): ), None, ) - pp( schema.collected_sql ) + + if not options['dry_run']: + with db.connection.schema_editor(collect_sql=False) as schema_editor: + for sql in schema.collected_sql: + schema_editor.execute(sql, None) + else: + for sql in schema.collected_sql: + self.stdout.write(sql) + # All done log.info('Done') From a59fb8bc26355839d70402398a405a48b13c68ce Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:13:23 -0400 Subject: [PATCH 282/454] The SQL output needs to be returned --- tubesync/sync/management/commands/fix-mariadb.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 7f19ff97..5fb9d4f0 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -242,13 +242,12 @@ class Command(BaseCommand): None, ) - if not options['dry_run']: + if options['dry_run']: + return '\n'.join(schema.collected_sql) + else: with db.connection.schema_editor(collect_sql=False) as schema_editor: for sql in schema.collected_sql: schema_editor.execute(sql, None) - else: - for sql in schema.collected_sql: - self.stdout.write(sql) # All done From 621a5787879e633bd74b851e4012700a41eb3f46 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:25:29 -0400 Subject: [PATCH 283/454] Log the `Done` before returning --- tubesync/sync/management/commands/fix-mariadb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 5fb9d4f0..4c515039 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -243,6 +243,7 @@ class Command(BaseCommand): ) if options['dry_run']: + log.info('Done') return '\n'.join(schema.collected_sql) else: with db.connection.schema_editor(collect_sql=False) as schema_editor: From 305d27a7af4c2f9c12cc66869aac365e02a119bf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 13:35:51 -0400 Subject: [PATCH 284/454] Try using `foreign_key_checks` --- tubesync/sync/management/commands/fix-mariadb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 4c515039..d9b608ec 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -181,7 +181,8 @@ class Command(BaseCommand): deferrable='', ) - schema.execute(remove_fk, None) + schema.execute('SET foreign_key_checks=0', None) + #schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( table=media_table_str, @@ -212,7 +213,8 @@ class Command(BaseCommand): ), None, ) - schema.execute(add_fk, None) + #schema.execute(add_fk, None) + schema.execute('SET foreign_key_checks=1', None) if table_names: From a8ee30192cf117e6fe55e828fdfe688407aef5c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 14:06:34 -0400 Subject: [PATCH 285/454] Remove `pprint` as it is now unused --- tubesync/sync/management/commands/fix-mariadb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index d9b608ec..c7d176e6 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -1,6 +1,5 @@ from django import db from io import BytesIO, TextIOWrapper -from pprint import pp from django.utils.translation import gettext_lazy from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError From c3b5a25b9d51e1b095dbea301fd14f38f5c1a1ba Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 21:36:43 -0400 Subject: [PATCH 286/454] Alter `sync_source` first --- .../sync/management/commands/fix-mariadb.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index c7d176e6..45098b3e 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -182,6 +182,16 @@ class Command(BaseCommand): schema.execute('SET foreign_key_checks=0', None) #schema.execute(remove_fk, None) + schema.execute( + schema.sql_alter_column % dict( + table=source_table_str, + changes=schema.sql_alter_column_not_null % dict( + type=uuid_type_str, + column=uuid_column_str, + ), + ), + None, + ) schema.execute( schema.sql_alter_column % dict( table=media_table_str, @@ -202,16 +212,6 @@ class Command(BaseCommand): ), None, ) - schema.execute( - schema.sql_alter_column % dict( - table=source_table_str, - changes=schema.sql_alter_column_not_null % dict( - type=uuid_type_str, - column=uuid_column_str, - ), - ), - None, - ) #schema.execute(add_fk, None) schema.execute('SET foreign_key_checks=1', None) From 0fb7520a2b022bad451869def2d80188570bf0c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 4 May 2025 22:25:37 -0400 Subject: [PATCH 287/454] Go back to removing then adding the foreign key --- tubesync/sync/management/commands/fix-mariadb.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 45098b3e..c3f2287a 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -180,11 +180,9 @@ class Command(BaseCommand): deferrable='', ) - schema.execute('SET foreign_key_checks=0', None) - #schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( - table=source_table_str, + table=media_table_str, changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, @@ -192,9 +190,10 @@ class Command(BaseCommand): ), None, ) + schema.execute(remove_fk, None) schema.execute( schema.sql_alter_column % dict( - table=media_table_str, + table=source_table_str, changes=schema.sql_alter_column_not_null % dict( type=uuid_type_str, column=uuid_column_str, @@ -212,8 +211,7 @@ class Command(BaseCommand): ), None, ) - #schema.execute(add_fk, None) - schema.execute('SET foreign_key_checks=1', None) + schema.execute(add_fk, None) if table_names: From c593933bfa38a2e21fee35424113229c10d49b95 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 5 May 2025 05:31:17 -0400 Subject: [PATCH 288/454] Add more details about saving database entries --- docs/other-database-backends.md | 39 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index 4f90d3ab..dbccb579 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -18,22 +18,49 @@ reset your database. If you are comfortable with Django you can export and re-im existing database data with: ```bash -$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json +$ docker exec -t tubesync \ + python3 /app/manage.py \ + dumpdata --format jsonl \ + --exclude background_task \ + --output /downloads/tubesync-database-backup.jsonl.xz ``` -Then change you database backend over, then use +Writing the compressed backup file to your `/downloads/` makes sense, as long as that directory is still available after destroying the current container. +If you have a configuration where that file will be deleted, choose a different place to store the output (perhaps `/config/`, if it has sufficient storage available) and place the file there instead. + +You can also copy the file from the container to the local filesystem (`/tmp/` in this example) with: ```bash -$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json +$ docker cp \ + tubesync:/downloads/tubesync-database-backup.jsonl.xz \ + /tmp/ +``` + +If you use `-` as the destination, then `docker cp` provides a `tar` archive. + +After you have changed your database backend over, then use: + +```bash +$ docker exec -t tubesync \ + python3 /app/manage.py \ + loaddata /downloads/tubesync-database-backup.jsonl.xz +``` + +Or, if you only have the copy in `/tmp/`, then you would use: +```bash +$ xzcat /tmp/tubesync-database-backup.jsonl.xz | \ + docker exec -i tubesync \ + python3 /app/manage.py \ + loaddata --format=jsonl - ``` As detailed in the Django documentation: -https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata +https://docs.djangoproject.com/en/5.1/ref/django-admin/#dumpdata and: -https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata +https://docs.djangoproject.com/en/5.1/ref/django-admin/#loaddata Further instructions are beyond the scope of TubeSync documenation and you should refer to Django documentation for more details. @@ -94,7 +121,7 @@ the DB for the performance benefits, a configuration like this would be enough: ``` tubesync-db: - image: postgres:15.2 + image: postgres:17 container_name: tubesync-db restart: unless-stopped volumes: From e90721722b3f3b3c8936893094f11154358dec5a Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 5 May 2025 06:06:38 -0400 Subject: [PATCH 289/454] Let `missing_pot` formats download after testing --- tubesync/sync/youtube.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 55046c81..4bf4f392 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -356,6 +356,7 @@ def download_media( 'sleep_interval': 10, 'max_sleep_interval': min(20*60, max(60, settings.DOWNLOAD_MEDIA_DELAY)), 'sleep_interval_requests': 1 + (2 * settings.BACKGROUND_TASK_ASYNC_THREADS), + 'extractor_args': opts.get('extractor_args', dict()), 'paths': opts.get('paths', dict()), 'postprocessor_args': opts.get('postprocessor_args', dict()), 'postprocessor_hooks': opts.get('postprocessor_hooks', list()), @@ -379,6 +380,18 @@ def download_media( 'temp': str(temp_dir_path), }) + # Allow download of formats that tested good with 'missing_pot' + youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict()) + formats_list = youtube_ea_dict.get('formats', list()) + if 'missing_pot' not in formats_list: + formats_list += ('missing_pot',) + youtube_ea_dict.update({ + 'formats': formats_list, + }) + ytopts['extractor_args'].update({ + 'youtube': youtube_ea_dict, + }) + postprocessor_hook_func = postprocessor_hook.get('function', None) if postprocessor_hook_func: ytopts['postprocessor_hooks'].append(postprocessor_hook_func) From 024ced3696f520185a032175b23f14dfa22f49a1 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 11:44:39 -0400 Subject: [PATCH 290/454] Remux video for combined format download --- tubesync/sync/youtube.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 55046c81..14c7f06b 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -336,12 +336,15 @@ def download_media( ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds + else: + pp_opts.remuxvideo = extension ytopts = { 'format': media_format, 'final_ext': extension, 'merge_output_format': extension, 'outtmpl': os.path.basename(output_file), + 'remuxvideo': pp_opts.remuxvideo, 'quiet': False if settings.DEBUG else True, 'verbose': True if settings.DEBUG else False, 'noprogress': None if settings.DEBUG else True, From 27ceecadc85616a20b723cbf7e8f209ab78f13ff Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 11:49:18 -0400 Subject: [PATCH 291/454] Check `media_format` for `+` --- tubesync/sync/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 14c7f06b..46840338 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -336,7 +336,7 @@ def download_media( ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds - else: + elif '+' not in media_format: pp_opts.remuxvideo = extension ytopts = { From a2722ee18866b198e5ee3d9569d2fc201835dca4 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 18:01:45 -0400 Subject: [PATCH 292/454] Use `POSTGRES_DB` instead of `init.sql` --- docs/other-database-backends.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index dbccb579..d623ddd5 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -125,9 +125,9 @@ the DB for the performance benefits, a configuration like this would be enough: container_name: tubesync-db restart: unless-stopped volumes: - - //init.sql:/docker-entrypoint-initdb.d/init.sql - //tubesync-db:/var/lib/postgresql/data environment: + - POSTGRES_DB=tubesync - POSTGRES_USER=postgres - POSTGRES_PASSWORD=testpassword @@ -145,15 +145,3 @@ the DB for the performance benefits, a configuration like this would be enough: depends_on: - tubesync-db ``` - -Note that an `init.sql` file is needed to initialize the `tubesync` -database before it can be written to. This file should contain: - -``` -CREATE DATABASE tubesync; -``` - - -Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it -to be executed on first startup of the container. See the `tubesync-db` -volume mapping above for how to do this. From ed90a238308ff0648dbac1f9354657e12d1835b7 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 6 May 2025 19:34:28 -0400 Subject: [PATCH 293/454] Stop services before database steps --- docs/other-database-backends.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index dbccb579..3d35ece2 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -18,6 +18,14 @@ reset your database. If you are comfortable with Django you can export and re-im existing database data with: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Backup the database into a compressed file $ docker exec -t tubesync \ python3 /app/manage.py \ dumpdata --format jsonl \ @@ -41,6 +49,14 @@ If you use `-` as the destination, then `docker cp` provides a `tar` archive. After you have changed your database backend over, then use: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Load fixture file into the database $ docker exec -t tubesync \ python3 /app/manage.py \ loaddata /downloads/tubesync-database-backup.jsonl.xz @@ -48,6 +64,14 @@ $ docker exec -t tubesync \ Or, if you only have the copy in `/tmp/`, then you would use: ```bash +# Stop services +$ docker exec -t tubesync \ + bash -c 'for svc in \ + /run/service/{gunicorn,tubesync*-worker} ; \ +do \ + /command/s6-svc -wd -D "${svc}" ; \ +done' +# Load fixture data from standard input into the database $ xzcat /tmp/tubesync-database-backup.jsonl.xz | \ docker exec -i tubesync \ python3 /app/manage.py \ From dab458c9f766c0583a1452836b6bf72b8f1c2f3b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 15:24:23 -0400 Subject: [PATCH 294/454] Add `music.youtube.com` to the YouTube domains --- tubesync/sync/choices.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py index 25dd762a..6412ad14 100644 --- a/tubesync/sync/choices.py +++ b/tubesync/sync/choices.py @@ -8,6 +8,7 @@ DOMAINS = dict({ 'youtube': frozenset({ 'youtube.com', 'm.youtube.com', + 'music.youtube.com', 'www.youtube.com', }), }) From e6a6552c3c3edcb0b8704ef73758ab3bdcfb2752 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 20:56:51 -0400 Subject: [PATCH 295/454] Add a start link to scheduled tasks --- tubesync/sync/templates/sync/tasks.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 9cb9dfe1..86de0441 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -67,9 +67,10 @@
{% for task in scheduled %} - {{ task }}
+ {{ task }}
{% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }}{% endif %} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + {% endif %} {% empty %} There are no scheduled tasks on this page. From 01705f5f68e18e4dbad850bb4ecf696e2a4d5418 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 21:23:42 -0400 Subject: [PATCH 296/454] Add task scheduling URLs --- tubesync/sync/urls.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 9cec74ee..72ed241a 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -122,6 +122,18 @@ urlpatterns = [ name='tasks', ), + path( + 'task//schedule/now', + TasksView.as_view(), + name='run-task', + ), + + path( + 'task//schedule/', + TasksView.as_view(), + name='schedule-task', + ), + path( 'tasks-completed', CompletedTasksView.as_view(), From dc38c0a37a78d8b3bacbb36dca3f0d545827c15d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 7 May 2025 21:25:04 -0400 Subject: [PATCH 297/454] Match the correct URL --- tubesync/sync/templates/sync/tasks.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 86de0441..fa857b1a 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -69,7 +69,7 @@ {{ task }}
{% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} {% endif %} {% empty %} From 3b17f693464820cf75ca235a17d2b6247c839da2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 07:55:06 -0400 Subject: [PATCH 298/454] Add `TaskScheduleView` --- tubesync/sync/views.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 73fb3ddd..5c3e2cff 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -24,7 +24,7 @@ from common.utils import append_uri_params from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, - SkipMediaForm, EnableMediaForm, ResetTasksForm, + SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, ConfirmDeleteMediaServerForm) from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, @@ -1004,6 +1004,41 @@ class ResetTasks(FormView): return append_uri_params(url, {'message': 'reset'}) +class TaskScheduleView(FormView, SingleObjectMixin): + ''' + Confirm that the task should be re-scheduled. + ''' + + template_name = 'sync/task-schedule.html' + form_class = ScheduleTaskForm + model = Task + + def __init__(self, *args, **kwargs): + self.object = None + self.schedule = 0 + super().__init__(*args, **kwargs) + + def dispatch(self, request, *args, **kwargs): + self.object = self.get_object() + return super().dispatch(request, *args, **kwargs) + + def get_success_url(self): + return append_uri_params( + reverse_lazy('sync:tasks'), + dict( + message='scheduled', + pk=str(self.object.pk), + ), + ) + + def form_valid(self, form): + max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) + self.object.attempts = max_attempts // 2 + self.object.run_at = timezone.now() + timezone.timedelta(seconds=self.schedule) + self.object.save() + return super().form_valid(form) + + class MediaServersView(ListView): ''' List of media servers which have been added. From 317f2939919a45e9be6766d1fb64477045e6d50f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 07:58:56 -0400 Subject: [PATCH 299/454] Add `ScheduleTaskForm` --- tubesync/sync/forms.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 3d795a5f..cf73f8b4 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -44,10 +44,16 @@ class ResetTasksForm(forms.Form): pass +class ScheduleTaskForm(forms.Form): + + pass + + class ConfirmDeleteMediaServerForm(forms.Form): pass + _media_server_type_label = 'Jellyfin' class JellyfinMediaServerForm(forms.Form): From 8da507c9350bb17cc45ae1dcc165538c38430a2a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:15:51 -0400 Subject: [PATCH 300/454] Create task-schedule.html --- .../sync/templates/sync/task-schedule.html | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tubesync/sync/templates/sync/task-schedule.html diff --git a/tubesync/sync/templates/sync/task-schedule.html b/tubesync/sync/templates/sync/task-schedule.html new file mode 100644 index 00000000..8f387b4a --- /dev/null +++ b/tubesync/sync/templates/sync/task-schedule.html @@ -0,0 +1,34 @@ +{% extends 'base.html' %} + +{% block headtitle %}Schedule task{% endblock %} + +{% block content %} +
+
+

Schedule task

+

+ If you don't want to wait for the existing schedule to be triggered, + you can use this to change when the task will be scheduled to run. + It is not guaranteed to run at any exact time, because when a task + requests to run and when a slot to execute it, in the appropriate + queue and with the priority level assigned, is dependent on how long + other tasks are taking to complete the assigned work. +

+

+ This will change the time that the task is requesting to be run + to the current time, plus some number of seconds. +

+
+
+
+
+ {% csrf_token %} + {% include 'simpleform.html' with form=form %} +
+
+ +
+
+
+
+{% endblock %} From 30b77d182b1f3d0a150bcfceae4b0973dfa21a09 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:19:20 -0400 Subject: [PATCH 301/454] Use `TaskScheduleView` --- tubesync/sync/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 72ed241a..232c21dc 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -3,7 +3,7 @@ from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceVie SourceView, UpdateSourceView, DeleteSourceView, MediaView, MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView, MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks, - MediaServersView, AddMediaServerView, MediaServerView, + TaskScheduleView, MediaServersView, AddMediaServerView, MediaServerView, DeleteMediaServerView, UpdateMediaServerView) @@ -124,7 +124,7 @@ urlpatterns = [ path( 'task//schedule/now', - TasksView.as_view(), + TaskScheduleView.as_view(), name='run-task', ), From d8eca035acb89553495a99a2621d02d199412e1f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 08:22:34 -0400 Subject: [PATCH 302/454] Update urls.py --- tubesync/sync/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py index 232c21dc..2dffea5d 100644 --- a/tubesync/sync/urls.py +++ b/tubesync/sync/urls.py @@ -130,7 +130,7 @@ urlpatterns = [ path( 'task//schedule/', - TasksView.as_view(), + TaskScheduleView.as_view(), name='schedule-task', ), From 329c13625c8d3d960c396911b4d1cf7e2c730f31 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 09:28:49 -0400 Subject: [PATCH 303/454] Add a missing error message --- tubesync/sync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 73fb3ddd..bdedb3fe 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -168,6 +168,7 @@ class ValidateSourceView(FormView): template_name = 'sync/source-validate.html' form_class = ValidateSourceForm errors = { + 'invalid_source': _('Invalid type for the source.'), 'invalid_url': _('Invalid URL, the URL must for a "{item}" must be in ' 'the format of "{example}". The error was: {error}.'), } From 4f685527132c3c8d53ed5c3a56bc0c90e8c1a1ef Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 09:50:01 -0400 Subject: [PATCH 304/454] Provide `when` for the form and the template --- tubesync/sync/views.py | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 5c3e2cff..218d259e 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -20,6 +20,7 @@ from django.utils.text import slugify from django.utils._os import safe_join from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from common.timestamp import timestamp_to_datetime from common.utils import append_uri_params from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer @@ -1012,16 +1013,42 @@ class TaskScheduleView(FormView, SingleObjectMixin): template_name = 'sync/task-schedule.html' form_class = ScheduleTaskForm model = Task + errors = dict( + invalid_when=_('The type ({}) was incorrect.'), + ) def __init__(self, *args, **kwargs): self.object = None - self.schedule = 0 + self.timestamp = None + self.when = None super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): self.object = self.get_object() + self.timestamp = kwargs.get('timestamp', '') + try: + self.timestamp = int(self.timestamp, 10) + except (TypeError, ValueError): + self.timestamp = None + else: + try: + self.when = timestamp_to_datetime(self.timestamp) + except AssertionError: + self.when = None + if self.when is None: + self.when = timezone.now() return super().dispatch(request, *args, **kwargs) + def get_initial(self): + initial = super().get_initial() + initial['when'] = self.when + return initial + + def get_context_data(self, *args, **kwargs): + data = super().get_context_data(*args, **kwargs) + data['when'] = self.when + return data + def get_success_url(self): return append_uri_params( reverse_lazy('sync:tasks'), @@ -1033,9 +1060,26 @@ class TaskScheduleView(FormView, SingleObjectMixin): def form_valid(self, form): max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) + now = timezone.now() + when = form.cleaned_data.get('when') + + if not isinstance(when, now.__class__): + form.add_error( + 'when', + ValidationError( + errors['invalid_when'].format( + type(when), + ), + ), + ) + + if form.errors: + return super().form_invalid(form) + self.object.attempts = max_attempts // 2 - self.object.run_at = timezone.now() + timezone.timedelta(seconds=self.schedule) + self.object.run_at = when self.object.save() + return super().form_valid(form) From 0e29a5bdbe84974f9dc136af8fd9028750d3b428 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:06:04 -0400 Subject: [PATCH 305/454] Add `when` to the form --- tubesync/sync/forms.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index cf73f8b4..c0bfd13f 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -46,7 +46,10 @@ class ResetTasksForm(forms.Form): class ScheduleTaskForm(forms.Form): - pass + when = forms.DateTimeField( + label=_('When the task should run'), + required=True, + ) class ConfirmDeleteMediaServerForm(forms.Form): From 90253ecab63f3660602ccd79ab4396500adac92a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:27:22 -0400 Subject: [PATCH 306/454] Display both links in the same item --- tubesync/sync/templates/sync/tasks.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index fa857b1a..1bf7b2c4 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -66,12 +66,17 @@

{% for task in scheduled %} - - {{ task }}
- {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} - Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} - {% endif %} - +
+ + {{ task }}
+ {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} + Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }} + + + + {% endif %} + +
{% empty %} There are no scheduled tasks on this page. {% endfor %} From dc3e320c11dc3a68c80eae9d4827ce8fd61ba177 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 10:36:03 -0400 Subject: [PATCH 307/454] Allow running tasks with errors also --- tubesync/sync/templates/sync/tasks.html | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 1bf7b2c4..33f6c40b 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -43,11 +43,16 @@

{% for task in errors %} - - {{ task }}, attempted {{ task.attempts }} time{{ task.attempts|pluralize }}
- Error: "{{ task.error_message }}"
+
+ + {{ task }}, attempted {{ task.attempts }} time{{ task.attempts|pluralize }}
+ Error: "{{ task.error_message }}"
+ Task will be retried at {{ task.run_at|date:'Y-m-d H:i:s' }} - + + + +
{% empty %} There are no tasks with errors on this page. {% endfor %} From f5fcfd2e1efd8e6410bdadf18d4f8ffd80edc1f4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 15:54:13 -0400 Subject: [PATCH 308/454] Log `OSError` when checking for existing media files --- tubesync/sync/signals.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 32b0b5f6..06c0e2fa 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -272,8 +272,15 @@ def media_post_save(sender, instance, created, **kwargs): thumbnail_url, verbose_name=verbose_name.format(instance.name), ) + media_file_exists = False + try: + media_file_exists |= instance.media_file_exists + media_file_exists |= instance.filepath.exists() + except OSError as e: + log.exception(e) + pass # If the media has not yet been downloaded schedule it to be downloaded - if not (instance.media_file_exists or instance.filepath.exists() or existing_media_download_task): + if not (media_file_exists or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. if instance.can_download and instance.downloaded: skip_changed = True != instance.skip From 266218f8163ed733400801c7b041292d8ad2c008 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:48:43 -0400 Subject: [PATCH 309/454] Rename tubesync/sync/models.py to tubesync/sync/models/__init__.py --- tubesync/sync/{models.py => models/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/{models.py => models/__init__.py} (100%) diff --git a/tubesync/sync/models.py b/tubesync/sync/models/__init__.py similarity index 100% rename from tubesync/sync/models.py rename to tubesync/sync/models/__init__.py From 40b7ce1d2eaf706061a3d535a5ff17b4ee427804 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:51:22 -0400 Subject: [PATCH 310/454] Create media_server.py --- tubesync/sync/models/media_server.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/media_server.py diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/media_server.py @@ -0,0 +1 @@ + From a8b811abcdb30fae8c8c82037e150768a2cb067f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:54:08 -0400 Subject: [PATCH 311/454] Create metadata_format.py --- tubesync/sync/models/metadata_format.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/metadata_format.py diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/metadata_format.py @@ -0,0 +1 @@ + From e764e43e6d20480b8dd17032d9f4376751c36a60 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:55:45 -0400 Subject: [PATCH 312/454] Create metadata.py --- tubesync/sync/models/metadata.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/metadata.py diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/metadata.py @@ -0,0 +1 @@ + From e33cf5220930a7917a546b2ef4bf45d56f95da6a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:56:11 -0400 Subject: [PATCH 313/454] Create source.py --- tubesync/sync/models/source.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/source.py diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/source.py @@ -0,0 +1 @@ + From 6717f1a58b66d41a8d10385c081b1fb4eac0d380 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 20:57:35 -0400 Subject: [PATCH 314/454] Create media.py --- tubesync/sync/models/media.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tubesync/sync/models/media.py diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tubesync/sync/models/media.py @@ -0,0 +1 @@ + From 182836aa990e2839bd236cbfd849a813068397c7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 21:33:55 -0400 Subject: [PATCH 315/454] Update media_server.py --- tubesync/sync/models/media_server.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index 8b137891..5e5e8c30 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1 +1,89 @@ +import json +from django import db +from django.utils.translation import gettext_lazy as _ +from ..choices import Val, MediaServerType + +class MediaServer(db.models.Model): + ''' + A remote media server, such as a Plex server. + ''' + + ICONS = { + Val(MediaServerType.JELLYFIN): '', + Val(MediaServerType.PLEX): '', + } + HANDLERS = MediaServerType.handlers_dict() + + server_type = db.models.CharField( + _('server type'), + max_length=1, + db_index=True, + choices=MediaServerType.choices, + default=MediaServerType.PLEX, + help_text=_('Server type'), + ) + host = db.models.CharField( + _('host'), + db_index=True, + max_length=200, + help_text=_('Hostname or IP address of the media server'), + ) + port = db.models.PositiveIntegerField( + _('port'), + db_index=True, + help_text=_('Port number of the media server'), + ) + use_https = db.models.BooleanField( + _('use https'), + default=False, + help_text=_('Connect to the media server over HTTPS'), + ) + verify_https = db.models.BooleanField( + _('verify https'), + default=True, + help_text=_('If connecting over HTTPS, verify the SSL certificate is valid'), + ) + options = db.models.JSONField( + _('options'), + blank=False, + null=True, + help_text=_('JSON encoded options for the media server'), + ) + + def __str__(self): + return f'{self.get_server_type_display()} server at {self.url}' + + class Meta: + verbose_name = _('Media Server') + verbose_name_plural = _('Media Servers') + unique_together = ( + ('host', 'port'), + ) + + @property + def url(self): + scheme = 'https' if self.use_https else 'http' + return f'{scheme}://{self.host.strip()}:{self.port}' + + @property + def icon(self): + return self.ICONS.get(self.server_type) + + @property + def handler(self): + handler_class = self.HANDLERS.get(self.server_type) + return handler_class(self) + + @property + def loaded_options(self): + return self.options or dict() + + def validate(self): + return self.handler.validate() + + def update(self): + return self.handler.update() + + def get_help_html(self): + return self.handler.HELP From 83ecb14b88e6447560a81a41708c33ab1945b9bb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 21:41:29 -0400 Subject: [PATCH 316/454] Update media_server.py --- tubesync/sync/models/media_server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index 5e5e8c30..bb9424c4 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1,4 +1,3 @@ -import json from django import db from django.utils.translation import gettext_lazy as _ from ..choices import Val, MediaServerType @@ -48,7 +47,7 @@ class MediaServer(db.models.Model): _('options'), blank=False, null=True, - help_text=_('JSON encoded options for the media server'), + help_text=_('Options for the media server'), ) def __str__(self): From cfe2e30f17cbb16c4a06bd0a4bf58c2c6bd4bdff Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:02:06 -0400 Subject: [PATCH 317/454] Update views.py --- tubesync/sync/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 692675a0..4d376165 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1146,14 +1146,14 @@ class AddMediaServerView(FormView): def form_valid(self, form): # Assign mandatory fields, bundle other fields into options mediaserver = MediaServer(server_type=self.server_type) - options = {} + options = dict() model_fields = [field.name for field in MediaServer._meta.fields] for field_name, field_value in form.cleaned_data.items(): if field_name in model_fields: setattr(mediaserver, field_name, field_value) else: options[field_name] = field_value - mediaserver.options = json.dumps(options) + mediaserver.options = options # Test the media server details are valid try: mediaserver.validate() @@ -1260,21 +1260,21 @@ class UpdateMediaServerView(FormView, SingleObjectMixin): for field in self.object._meta.fields: if field.name in self.form_class.declared_fields: initial[field.name] = getattr(self.object, field.name) - for option_key, option_val in self.object.loaded_options.items(): + for option_key, option_val in self.object.options.items(): if option_key in self.form_class.declared_fields: initial[option_key] = option_val return initial def form_valid(self, form): # Assign mandatory fields, bundle other fields into options - options = {} + options = dict() model_fields = [field.name for field in MediaServer._meta.fields] for field_name, field_value in form.cleaned_data.items(): if field_name in model_fields: setattr(self.object, field_name, field_value) else: options[field_name] = field_value - self.object.options = json.dumps(options) + self.object.options = options # Test the media server details are valid try: self.object.validate() From a12363006980f09eb25206e605c833780e5c85ab Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:05:36 -0400 Subject: [PATCH 318/454] Update views.py --- tubesync/sync/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 4d376165..762f770e 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1,6 +1,5 @@ import glob import os -import json from base64 import b64decode import pathlib import sys From 84298ab5e5f3ec8d1ac6cf393c3e4d7f89c7005f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:12:24 -0400 Subject: [PATCH 319/454] Update mediaserver.html --- tubesync/sync/templates/sync/mediaserver.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/mediaserver.html b/tubesync/sync/templates/sync/mediaserver.html index 23546eba..2626d77c 100644 --- a/tubesync/sync/templates/sync/mediaserver.html +++ b/tubesync/sync/templates/sync/mediaserver.html @@ -28,7 +28,7 @@ Verify HTTPS Verify HTTPS
{% if mediaserver.verify_https %}{% else %}{% endif %} - {% for name, value in mediaserver.loaded_options.items %} + {% for name, value in mediaserver.options.items %} {{ name|title }} {{ name|title }}
{% if name in private_options %}{{ value|truncatechars:6 }} (hidden){% else %}{{ value }}{% endif %} From ea4733f725d0cebf0de2291f44432c8535deb851 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:31:12 -0400 Subject: [PATCH 320/454] Update mediaservers.py --- tubesync/sync/mediaservers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index e0f9e7e7..ceab239f 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -29,7 +29,7 @@ class MediaServer: def make_request_args(self, uri='/', token_header=None, headers={}, token_param=None, params={}): base_parts = urlsplit(self.object.url) if self.token is None: - self.token = self.object.loaded_options['token'] or None + self.token = self.object.options['token'] or None if token_header and self.token: headers.update({token_header: self.token}) self.headers.update(headers) @@ -116,7 +116,7 @@ class PlexMediaServer(MediaServer): if port < 1 or port > 65535: raise ValidationError('Plex Media Server "port" must be between 1 ' 'and 65535') - options = self.object.loaded_options + options = self.object.options if 'token' not in options: raise ValidationError('Plex Media Server requires a "token"') token = options['token'].strip() @@ -183,7 +183,7 @@ class PlexMediaServer(MediaServer): def update(self): # For each section / library ID pop off a request to refresh it - libraries = self.object.loaded_options.get('libraries', '') + libraries = self.object.options.get('libraries', '') for library_id in libraries.split(','): library_id = library_id.strip() uri = f'/library/sections/{library_id}/refresh' @@ -258,7 +258,7 @@ class JellyfinMediaServer(MediaServer): except (TypeError, ValueError): raise ValidationError('Jellyfin Media Server "port" must be an integer') - options = self.object.loaded_options + options = self.object.options if 'token' not in options: raise ValidationError('Jellyfin Media Server requires a "token"') if 'libraries' not in options: @@ -302,7 +302,7 @@ class JellyfinMediaServer(MediaServer): return True def update(self): - libraries = self.object.loaded_options.get('libraries', '').split(',') + libraries = self.object.options.get('libraries', '').split(',') for library_id in map(str.strip, libraries): uri = f'/Items/{library_id}/Refresh' response = self.make_request(uri, method='POST') From 5c987f1d582c4a222077c1a09a632f8c42ee8332 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:43:26 -0400 Subject: [PATCH 321/454] Update __init__.py --- tubesync/sync/models/__init__.py | 99 ++------------------------------ 1 file changed, 6 insertions(+), 93 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index a5e3d675..d0029c53 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -21,20 +21,21 @@ from common.logger import log from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) -from .youtube import ( get_media_info as get_youtube_media_info, +from ..youtube import ( get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) -from .utils import (seconds_to_timestr, parse_media_format, filter_response, +from ..utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote, multi_key_sort) -from .matching import ( get_best_combined_format, get_best_audio_format, +from ..matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) -from .fields import CommaSepChoiceField -from .choices import ( Val, CapChoices, Fallback, FileExtension, +from ..fields import CommaSepChoiceField +from ..choices import ( Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) @@ -1998,91 +1999,3 @@ class MetadataFormat(models.Model): self.site, self.value.get('format') or self.value.get('format_id'), ) - - -class MediaServer(models.Model): - ''' - A remote media server, such as a Plex server. - ''' - - ICONS = { - Val(MediaServerType.JELLYFIN): '', - Val(MediaServerType.PLEX): '', - } - HANDLERS = MediaServerType.handlers_dict() - - server_type = models.CharField( - _('server type'), - max_length=1, - db_index=True, - choices=MediaServerType.choices, - default=MediaServerType.PLEX, - help_text=_('Server type') - ) - host = models.CharField( - _('host'), - db_index=True, - max_length=200, - help_text=_('Hostname or IP address of the media server') - ) - port = models.PositiveIntegerField( - _('port'), - db_index=True, - help_text=_('Port number of the media server') - ) - use_https = models.BooleanField( - _('use https'), - default=False, - help_text=_('Connect to the media server over HTTPS') - ) - verify_https = models.BooleanField( - _('verify https'), - default=True, - help_text=_('If connecting over HTTPS, verify the SSL certificate is valid') - ) - options = models.TextField( - _('options'), - blank=False, # valid JSON only - null=True, - help_text=_('JSON encoded options for the media server') - ) - - def __str__(self): - return f'{self.get_server_type_display()} server at {self.url}' - - class Meta: - verbose_name = _('Media Server') - verbose_name_plural = _('Media Servers') - unique_together = ( - ('host', 'port'), - ) - - @property - def url(self): - scheme = 'https' if self.use_https else 'http' - return f'{scheme}://{self.host.strip()}:{self.port}' - - @property - def icon(self): - return self.ICONS.get(self.server_type) - - @property - def handler(self): - handler_class = self.HANDLERS.get(self.server_type) - return handler_class(self) - - @property - def loaded_options(self): - try: - return json.loads(self.options) - except Exception as e: - return {} - - def validate(self): - return self.handler.validate() - - def update(self): - return self.handler.update() - - def get_help_html(self): - return self.handler.HELP From ded3602d541c5ea93add1d0da07615c0336fc00b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 22:46:53 -0400 Subject: [PATCH 322/454] Update media_server.py --- tubesync/sync/models/media_server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index bb9424c4..a8ecefc6 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -74,10 +74,6 @@ class MediaServer(db.models.Model): handler_class = self.HANDLERS.get(self.server_type) return handler_class(self) - @property - def loaded_options(self): - return self.options or dict() - def validate(self): return self.handler.validate() From 5ab4e41b8d5918c9b01c60d790af7b902d0562fa Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:02:08 -0400 Subject: [PATCH 323/454] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 74 +++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index 8b137891..d44ca2a3 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -1 +1,75 @@ +import uuid +from django import db +from django.utils.translation import gettext_lazy as _ +#from .metadata import Metadata +from . import Metadata, JSONEncoder +class MetadataFormat(db.models.Model): + ''' + A format from the Metadata for an indexed `Media` item. + ''' + class Meta: + db_table = f'{Metadata._meta.db_table}_format' + verbose_name = _('Format from Media Metadata') + verbose_name_plural = _('Formats from Media Metadata') + unique_together = ( + ('metadata', 'site', 'key', 'number'), + ) + ordering = ['site', 'key', 'number'] + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the format'), + ) + metadata = db.models.ForeignKey( + Metadata, + # on_delete=models.DO_NOTHING, + on_delete=db.models.CASCADE, + related_name='format', + help_text=_('Metadata the format belongs to'), + null=False, + ) + site = db.models.CharField( + _('site'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='Youtube', + help_text=_('Site from which the format is available'), + ) + key = db.models.CharField( + _('key'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='', + help_text=_('Media identifier at the site from which this format is available'), + ) + number = db.models.PositiveIntegerField( + _('number'), + blank=False, + null=False, + help_text=_('Ordering number for this format'), + ) + value = db.models.JSONField( + _('value'), + encoder=JSONEncoder, + null=False, + default=dict, + help_text=_('JSON metadata format object'), + ) + + + def __str__(self): + template = '#{:n} "{}" from {}: {}' + return template.format( + self.number, + self.key, + self.site, + self.value.get('format') or self.value.get('format_id'), + ) From a00d760f0448e6aa3c15b8907120025e8198fc4e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:07:12 -0400 Subject: [PATCH 324/454] Create json.py --- tubesync/common/json.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tubesync/common/json.py diff --git a/tubesync/common/json.py b/tubesync/common/json.py new file mode 100644 index 00000000..e8a22e1c --- /dev/null +++ b/tubesync/common/json.py @@ -0,0 +1,16 @@ +from django.core.serializers.json import DjangoJSONEncoder + + +class JSONEncoder(DjangoJSONEncoder): + item_separator = ',' + key_separator = ':' + + def default(self, obj): + try: + iterable = iter(obj) + except TypeError: + pass + else: + return list(iterable) + return super().default(obj) + From d962d57ce08b7ac69a5dba87f14614adf8896a96 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:10:14 -0400 Subject: [PATCH 325/454] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index d44ca2a3..f4bea7ed 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -1,8 +1,9 @@ import uuid +from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ #from .metadata import Metadata -from . import Metadata, JSONEncoder +from . import Metadata class MetadataFormat(db.models.Model): ''' From 705169746be071a0b6d98ea7c455ec9f39194b62 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:12:28 -0400 Subject: [PATCH 326/454] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 8b137891..06340ed6 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -1 +1 @@ - +from . import Metadata From ac0eaa3ec5871d75edbf513bf252147822c49adf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:12:58 -0400 Subject: [PATCH 327/454] Update metadata_format.py --- tubesync/sync/models/metadata_format.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/metadata_format.py b/tubesync/sync/models/metadata_format.py index f4bea7ed..c116575b 100644 --- a/tubesync/sync/models/metadata_format.py +++ b/tubesync/sync/models/metadata_format.py @@ -2,8 +2,7 @@ import uuid from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ -#from .metadata import Metadata -from . import Metadata +from .metadata import Metadata class MetadataFormat(db.models.Model): ''' From 108f7169e33ebd23eb1602fedb6e768ade3c9704 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:17:38 -0400 Subject: [PATCH 328/454] Update __init__.py --- tubesync/sync/models/__init__.py | 72 +------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index d0029c53..8218b0cf 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -18,6 +18,7 @@ from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log +from common.json import JSONEncoder from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) @@ -35,6 +36,7 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .metadata_format import MetadataFormat from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') @@ -1929,73 +1931,3 @@ class Metadata(models.Model): return self.with_formats - -class MetadataFormat(models.Model): - ''' - A format from the Metadata for an indexed `Media` item. - ''' - class Meta: - db_table = f'{Metadata._meta.db_table}_format' - verbose_name = _('Format from Media Metadata') - verbose_name_plural = _('Formats from Media Metadata') - unique_together = ( - ('metadata', 'site', 'key', 'number'), - ) - ordering = ['site', 'key', 'number'] - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the format'), - ) - metadata = models.ForeignKey( - Metadata, - # on_delete=models.DO_NOTHING, - on_delete=models.CASCADE, - related_name='format', - help_text=_('Metadata the format belongs to'), - null=False, - ) - site = models.CharField( - _('site'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='Youtube', - help_text=_('Site from which the format is available'), - ) - key = models.CharField( - _('key'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='', - help_text=_('Media identifier at the site from which this format is available'), - ) - number = models.PositiveIntegerField( - _('number'), - blank=False, - null=False, - help_text=_('Ordering number for this format') - ) - value = models.JSONField( - _('value'), - encoder=JSONEncoder, - null=False, - default=dict, - help_text=_('JSON metadata format object'), - ) - - - def __str__(self): - template = '#{:n} "{}" from {}: {}' - return template.format( - self.number, - self.key, - self.site, - self.value.get('format') or self.value.get('format_id'), - ) From 9e66bf5151d3164ab77cf2d3d57afc702eea7786 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:37:27 -0400 Subject: [PATCH 329/454] Update metadata.py --- tubesync/sync/models/metadata.py | 155 ++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 06340ed6..de09ca56 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -1 +1,154 @@ -from . import Metadata +import uuid +from common.json import JSONEncoder +from common.timestamp import timestamp_to_datetime +from common.utils import django_queryset_generator as qs_gen +from django import db +from django.utils.translation import gettext_lazy as _ +#from .media import Media +from . import Media + + +class Metadata(db.models.Model): + ''' + Metadata for an indexed `Media` item. + ''' + class Meta: + db_table = 'sync_media_metadata' + verbose_name = _('Metadata about Media') + verbose_name_plural = _('Metadata about Media') + unique_together = ( + ('media', 'site', 'key'), + ) + get_latest_by = ["-retrieved", "-created"] + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the metadata'), + ) + media = db.models.OneToOneField( + Media, + # on_delete=models.DO_NOTHING, + on_delete=db.models.SET_NULL, + related_name='new_metadata', + help_text=_('Media the metadata belongs to'), + null=True, + parent_link=False, + ) + site = db.models.CharField( + _('site'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='Youtube', + help_text=_('Site from which the metadata was retrieved'), + ) + key = db.models.CharField( + _('key'), + max_length=256, + blank=True, + db_index=True, + null=False, + default='', + help_text=_('Media identifier at the site from which the metadata was retrieved'), + ) + created = db.models.DateTimeField( + _('created'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was created'), + ) + retrieved = db.models.DateTimeField( + _('retrieved'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the metadata was retrieved'), + ) + uploaded = db.models.DateTimeField( + _('uploaded'), + db_index=True, + null=True, + help_text=_('Date and time the media was uploaded'), + ) + published = db.models.DateTimeField( + _('published'), + db_index=True, + null=True, + help_text=_('Date and time the media was published'), + ) + value = db.models.JSONField( + _('value'), + encoder=JSONEncoder, + null=False, + default=dict, + help_text=_('JSON metadata object'), + ) + + + def __str__(self): + template = '"{}" from {} at: {}' + return template.format( + self.key, + self.site, + self.retrieved.isoformat(timespec='seconds'), + ) + + @db.transaction.atomic(durable=False) + def ingest_formats(self, formats=list(), /): + number = 0 + for number, format in enumerate(formats, start=1): + mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) + mdf.value = format + mdf.save() + if number > 0: + # delete any numbers we did not overwrite or create + self.format.filter(site=self.site, key=self.key, number__gt=number).delete() + + @property + def with_formats(self): + formats = self.format.all().order_by('number') + formats_list = [ f.value for f in qs_gen(formats) ] + metadata = self.value.copy() + metadata.update(dict(formats=formats_list)) + return metadata + + @db.transaction.atomic(durable=False) + def ingest_metadata(self, data): + assert isinstance(data, dict), type(data) + + try: + self.retrieved = timestamp_to_datetime( + self.media.get_metadata_first_value( + 'epoch', + arg_dict=data, + ) + ) or self.created + except AssertionError: + self.retrieved = self.created + + try: + self.published = timestamp_to_datetime( + self.media.get_metadata_first_value( + ('release_timestamp', 'timestamp',), + arg_dict=data, + ) + ) or self.media.published + except AssertionError: + self.published = self.media.published + + self.value = data.copy() # try not to have side-effects for the caller + formats_key = self.media.get_metadata_field('formats') + formats = self.value.pop(formats_key, list()) + self.uploaded = min( + self.published, + self.retrieved, + self.media.created, + ) + self.save() + self.ingest_formats(formats) + + return self.with_formats + From 0d9542163bf1d64f1b65b36886a071a645a911c3 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:43:16 -0400 Subject: [PATCH 330/454] Update __init__.py --- tubesync/sync/models/__init__.py | 161 +------------------------------ 1 file changed, 1 insertion(+), 160 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 8218b0cf..7ff69e97 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -36,26 +36,13 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) +from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) -class JSONEncoder(DjangoJSONEncoder): - item_separator = ',' - key_separator = ':' - - def default(self, obj): - try: - iterable = iter(obj) - except TypeError: - pass - else: - return list(iterable) - return super().default(obj) - - class Source(models.Model): ''' A Source is a source of media. Currently, this is either a YouTube channel @@ -1785,149 +1772,3 @@ class Media(models.Model): except OSError as e: pass - -class Metadata(models.Model): - ''' - Metadata for an indexed `Media` item. - ''' - class Meta: - db_table = 'sync_media_metadata' - verbose_name = _('Metadata about Media') - verbose_name_plural = _('Metadata about Media') - unique_together = ( - ('media', 'site', 'key'), - ) - get_latest_by = ["-retrieved", "-created"] - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the metadata'), - ) - media = models.OneToOneField( - Media, - # on_delete=models.DO_NOTHING, - on_delete=models.SET_NULL, - related_name='new_metadata', - help_text=_('Media the metadata belongs to'), - null=True, - parent_link=False, - ) - site = models.CharField( - _('site'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='Youtube', - help_text=_('Site from which the metadata was retrieved'), - ) - key = models.CharField( - _('key'), - max_length=256, - blank=True, - db_index=True, - null=False, - default='', - help_text=_('Media identifier at the site from which the metadata was retrieved'), - ) - created = models.DateTimeField( - _('created'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the metadata was created'), - ) - retrieved = models.DateTimeField( - _('retrieved'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the metadata was retrieved'), - ) - uploaded = models.DateTimeField( - _('uploaded'), - db_index=True, - null=True, - help_text=_('Date and time the media was uploaded'), - ) - published = models.DateTimeField( - _('published'), - db_index=True, - null=True, - help_text=_('Date and time the media was published'), - ) - value = models.JSONField( - _('value'), - encoder=JSONEncoder, - null=False, - default=dict, - help_text=_('JSON metadata object'), - ) - - - def __str__(self): - template = '"{}" from {} at: {}' - return template.format( - self.key, - self.site, - self.retrieved.isoformat(timespec='seconds'), - ) - - @atomic(durable=False) - def ingest_formats(self, formats=list(), /): - number = 0 - for number, format in enumerate(formats, start=1): - mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number) - mdf.value = format - mdf.save() - if number > 0: - # delete any numbers we did not overwrite or create - self.format.filter(site=self.site, key=self.key, number__gt=number).delete() - - @property - def with_formats(self): - formats = self.format.all().order_by('number') - formats_list = [ f.value for f in qs_gen(formats) ] - metadata = self.value.copy() - metadata.update(dict(formats=formats_list)) - return metadata - - @atomic(durable=False) - def ingest_metadata(self, data): - assert isinstance(data, dict), type(data) - from common.timestamp import timestamp_to_datetime - - try: - self.retrieved = timestamp_to_datetime( - self.media.get_metadata_first_value( - 'epoch', - arg_dict=data, - ) - ) or self.created - except AssertionError: - self.retrieved = self.created - - try: - self.published = timestamp_to_datetime( - self.media.get_metadata_first_value( - ('release_timestamp', 'timestamp',), - arg_dict=data, - ) - ) or self.media.published - except AssertionError: - self.published = self.media.published - - self.value = data.copy() # try not to have side-effects for the caller - formats_key = self.media.get_metadata_field('formats') - formats = self.value.pop(formats_key, list()) - self.uploaded = min( - self.published, - self.retrieved, - self.media.created, - ) - self.save() - self.ingest_formats(formats) - - return self.with_formats - From a5afdc6a456b998120a8a9ea2883388ac9b5c000 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:46:09 -0400 Subject: [PATCH 331/454] Update metadata.py --- tubesync/sync/models/metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index de09ca56..17d214fb 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,8 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -#from .media import Media -from . import Media +from .media import Media class Metadata(db.models.Model): From 6cc19363257791682a7f6f74d9e4ebba2f9cc6ad Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:48:43 -0400 Subject: [PATCH 332/454] Rename __init__.py to legacy.py --- tubesync/sync/models/{__init__.py => legacy.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tubesync/sync/models/{__init__.py => legacy.py} (100%) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/legacy.py similarity index 100% rename from tubesync/sync/models/__init__.py rename to tubesync/sync/models/legacy.py From 2e59339c6bdeea272fe253a0fbc68023f6c42434 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:50:01 -0400 Subject: [PATCH 333/454] Create __init__.py --- tubesync/sync/models/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tubesync/sync/models/__init__.py diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py new file mode 100644 index 00000000..b144173f --- /dev/null +++ b/tubesync/sync/models/__init__.py @@ -0,0 +1,5 @@ +from .metadata import Metadata +from .metadata_format import MetadataFormat +from .media_server import MediaServer + +from .legacy import * From 0251b9a8aa1ab44363f718c99548df5640ed8bd8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:50:36 -0400 Subject: [PATCH 334/454] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 17d214fb..e062ea0c 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,7 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -from .media import Media +from .legacy import Media class Metadata(db.models.Model): From d4003416d20960b8d911c7aa9c59795351dbaac2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:53:33 -0400 Subject: [PATCH 335/454] Update legacy.py --- tubesync/sync/models/legacy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/legacy.py index 7ff69e97..a7facceb 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/legacy.py @@ -36,9 +36,6 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) -from .metadata import Metadata -from .metadata_format import MetadataFormat -from .media_server import MediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) From aad8709bb8c27acd6ddc918656c7ec542dd6f087 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 8 May 2025 23:56:38 -0400 Subject: [PATCH 336/454] Delete tubesync/sync/models/media.py --- tubesync/sync/models/media.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tubesync/sync/models/media.py diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py deleted file mode 100644 index 8b137891..00000000 --- a/tubesync/sync/models/media.py +++ /dev/null @@ -1 +0,0 @@ - From 9ceafe9fd2658a33415921d289bea9753e41ee68 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:21:26 -0400 Subject: [PATCH 337/454] Update source.py --- tubesync/sync/models/source.py | 546 +++++++++++++++++++++++++++++++++ 1 file changed, 546 insertions(+) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 8b137891..01a0d36e 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -1 +1,547 @@ +import os +import re +import uuid +from pathlib import Path +from django import db +from django.conf import settings +from django.core.exceptions import SuspiciousOperation +from django.utils import timezone +from django.utils.text import slugify +from django.utils.translation import gettext_lazy as _ +from ..choices import (Val, + SponsorBlock_Category, YouTube_SourceType, IndexSchedule, + CapChoices, FilterSeconds, FileExtension, + SourceResolution, SourceResolutionInteger, + YouTube_VideoCodec, YouTube_AudioCodec, +) +from ..fields import CommaSepChoiceField +from ..youtube import ( + get_media_info as get_youtube_media_info, + get_channel_image_info as get_youtube_channel_image_info, +) +from .legacy import _srctype_dict, media_file_storage + + +class Source(db.models.Model): + ''' + A Source is a source of media. Currently, this is either a YouTube channel + or a YouTube playlist. + ''' + + sponsorblock_categories = CommaSepChoiceField( + _(''), + max_length=128, + possible_choices=SponsorBlock_Category.choices, + all_choice='all', + allow_all=True, + all_label='(All Categories)', + default='all', + help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'), + ) + embed_metadata = db.models.BooleanField( + _('embed metadata'), + default=False, + help_text=_('Embed metadata from source into file'), + ) + embed_thumbnail = db.models.BooleanField( + _('embed thumbnail'), + default=False, + help_text=_('Embed thumbnail into the file'), + ) + enable_sponsorblock = db.models.BooleanField( + _('enable sponsorblock'), + default=True, + help_text=_('Use SponsorBlock?'), + ) + + # Fontawesome icons used for the source on the front end + ICONS = _srctype_dict('') + + # Format to use to display a URL for the source + URLS = dict(zip( + YouTube_SourceType.values, + ( + 'https://www.youtube.com/c/{key}', + 'https://www.youtube.com/channel/{key}', + 'https://www.youtube.com/playlist?list={key}', + ), + )) + + # Format used to create indexable URLs + INDEX_URLS = dict(zip( + YouTube_SourceType.values, + ( + 'https://www.youtube.com/c/{key}/{type}', + 'https://www.youtube.com/channel/{key}/{type}', + 'https://www.youtube.com/playlist?list={key}', + ), + )) + + # Callback functions to get a list of media from the source + INDEXERS = _srctype_dict(get_youtube_media_info) + + # Field names to find the media ID used as the key when storing media + KEY_FIELD = _srctype_dict('id') + + uuid = db.models.UUIDField( + _('uuid'), + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text=_('UUID of the source'), + ) + created = db.models.DateTimeField( + _('created'), + auto_now_add=True, + db_index=True, + help_text=_('Date and time the source was created'), + ) + last_crawl = db.models.DateTimeField( + _('last crawl'), + db_index=True, + null=True, + blank=True, + help_text=_('Date and time the source was last crawled'), + ) + source_type = db.models.CharField( + _('source type'), + max_length=1, + db_index=True, + choices=YouTube_SourceType.choices, + default=YouTube_SourceType.CHANNEL, + help_text=_('Source type'), + ) + key = db.models.CharField( + _('key'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Source key, such as exact YouTube channel name or playlist ID'), + ) + name = db.models.CharField( + _('name'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Friendly name for the source, used locally in TubeSync only'), + ) + directory = db.models.CharField( + _('directory'), + max_length=100, + db_index=True, + unique=True, + help_text=_('Directory name to save the media into'), + ) + media_format = db.models.CharField( + _('media format'), + max_length=200, + default=settings.MEDIA_FORMATSTR_DEFAULT, + help_text=_('File format to use for saving files, detailed options at bottom of page.'), + ) + index_schedule = db.models.IntegerField( + _('index schedule'), + choices=IndexSchedule.choices, + db_index=True, + default=IndexSchedule.EVERY_24_HOURS, + help_text=_('Schedule of how often to index the source for new media'), + ) + download_media = db.models.BooleanField( + _('download media'), + default=True, + help_text=_('Download media from this source, if not selected the source will only be indexed'), + ) + index_videos = db.models.BooleanField( + _('index videos'), + default=True, + help_text=_('Index video media from this source'), + ) + index_streams = db.models.BooleanField( + _('index streams'), + default=False, + help_text=_('Index live stream media from this source'), + ) + download_cap = db.models.IntegerField( + _('download cap'), + choices=CapChoices.choices, + default=CapChoices.CAP_NOCAP, + help_text=_('Do not download media older than this capped date'), + ) + delete_old_media = db.models.BooleanField( + _('delete old media'), + default=False, + help_text=_('Delete old media after "days to keep" days?'), + ) + days_to_keep = db.models.PositiveSmallIntegerField( + _('days to keep'), + default=14, + help_text=_( + 'If "delete old media" is ticked, the number of days after which ' + 'to automatically delete media' + ), + ) + filter_text = db.models.CharField( + _('filter string'), + max_length=200, + default='', + blank=True, + help_text=_('Regex compatible filter string for video titles'), + ) + filter_text_invert = db.models.BooleanField( + _('invert filter text matching'), + default=False, + help_text=_('Invert filter string regex match, skip any matching titles when selected'), + ) + filter_seconds = db.models.PositiveIntegerField( + _('filter seconds'), + blank=True, + null=True, + help_text=_('Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering'), + ) + filter_seconds_min = db.models.BooleanField( + _('filter seconds min/max'), + choices=FilterSeconds.choices, + default=Val(FilterSeconds.MIN), + help_text=_( + 'When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video ' + 'greater than maximum) video duration' + ), + ) + delete_removed_media = db.models.BooleanField( + _('delete removed media'), + default=False, + help_text=_('Delete media that is no longer on this playlist'), + ) + delete_files_on_disk = db.models.BooleanField( + _('delete files on disk'), + default=False, + help_text=_('Delete files on disk when they are removed from TubeSync'), + ) + source_resolution = db.models.CharField( + _('source resolution'), + max_length=8, + db_index=True, + choices=SourceResolution.choices, + default=SourceResolution.VIDEO_1080P, + help_text=_('Source resolution, desired video resolution to download'), + ) + source_vcodec = db.models.CharField( + _('source video codec'), + max_length=8, + db_index=True, + choices=YouTube_VideoCodec.choices, + default=YouTube_VideoCodec.VP9, + help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)'), + ) + source_acodec = db.models.CharField( + _('source audio codec'), + max_length=8, + db_index=True, + choices=YouTube_AudioCodec.choices, + default=YouTube_AudioCodec.OPUS, + help_text=_('Source audio codec, desired audio encoding format to download'), + ) + prefer_60fps = db.models.BooleanField( + _('prefer 60fps'), + default=True, + help_text=_('Where possible, prefer 60fps media for this source'), + ) + prefer_hdr = db.models.BooleanField( + _('prefer hdr'), + default=False, + help_text=_('Where possible, prefer HDR media for this source'), + ) + fallback = db.models.CharField( + _('fallback'), + max_length=1, + db_index=True, + choices=Fallback.choices, + default=Fallback.NEXT_BEST_HD, + help_text=_('What do do when media in your source resolution and codecs is not available'), + ) + copy_channel_images = db.models.BooleanField( + _('copy channel images'), + default=False, + help_text=_('Copy channel banner and avatar. These may be detected and used by some media servers'), + ) + copy_thumbnails = db.models.BooleanField( + _('copy thumbnails'), + default=False, + help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers'), + ) + write_nfo = db.models.BooleanField( + _('write nfo'), + default=False, + help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers'), + ) + write_json = db.models.BooleanField( + _('write json'), + default=False, + help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers'), + ) + has_failed = db.models.BooleanField( + _('has failed'), + default=False, + help_text=_('Source has failed to index media'), + ) + + write_subtitles = db.models.BooleanField( + _('write subtitles'), + default=False, + help_text=_('Download video subtitles'), + ) + + auto_subtitles = db.models.BooleanField( + _('accept auto-generated subs'), + default=False, + help_text=_('Accept auto-generated subtitles'), + ) + sub_langs = db.models.CharField( + _('subs langs'), + max_length=30, + default='en', + help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), + validators=[ + RegexValidator( + regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", + message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat'), + ), + ], + ) + + def __str__(self): + return self.name + + class Meta: + verbose_name = _('Source') + verbose_name_plural = _('Sources') + + @property + def icon(self): + return self.ICONS.get(self.source_type) + + @property + def slugname(self): + replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') + return slugify(replaced)[:80] + + def deactivate(self): + self.download_media = False + self.index_streams = False + self.index_videos = False + self.index_schedule = IndexSchedule.NEVER + self.save(update_fields={ + 'download_media', + 'index_streams', + 'index_videos', + 'index_schedule', + }) + + @property + def is_active(self): + active = ( + self.download_media or + self.index_streams or + self.index_videos + ) + return self.index_schedule and active + + @property + def is_audio(self): + return self.source_resolution == SourceResolution.AUDIO.value + + @property + def is_playlist(self): + return self.source_type == YouTube_SourceType.PLAYLIST.value + + @property + def is_video(self): + return not self.is_audio + + @property + def download_cap_date(self): + delta = self.download_cap + if delta > 0: + return timezone.now() - timezone.timedelta(seconds=delta) + else: + return False + + @property + def days_to_keep_date(self): + delta = self.days_to_keep + if delta > 0: + return timezone.now() - timezone.timedelta(days=delta) + else: + return False + + @property + def extension(self): + ''' + The extension is also used by youtube-dl to set the output container. As + it is possible to quite easily pick combinations of codecs and containers + which are invalid (e.g. OPUS audio in an MP4 container) just set this for + people. All video is set to mkv containers, audio-only is set to m4a or ogg + depending on audio codec. + ''' + if self.is_audio: + if self.source_acodec == Val(YouTube_AudioCodec.MP4A): + return Val(FileExtension.M4A) + elif self.source_acodec == Val(YouTube_AudioCodec.OPUS): + return Val(FileExtension.OGG) + else: + raise ValueError('Unable to choose audio extension, uknown acodec') + else: + return Val(FileExtension.MKV) + + @classmethod + def create_url(cls, source_type, key): + url = cls.URLS.get(source_type) + return url.format(key=key) + + @classmethod + def create_index_url(cls, source_type, key, type): + url = cls.INDEX_URLS.get(source_type) + return url.format(key=key, type=type) + + @property + def url(self): + return self.__class__.create_url(self.source_type, self.key) + + def get_index_url(self, type): + return self.__class__.create_index_url(self.source_type, self.key, type) + + @property + def format_summary(self): + if self.is_audio: + vc = 'none' + else: + vc = self.source_vcodec + ac = self.source_acodec + f = ' 60FPS' if self.is_video and self.prefer_60fps else '' + h = ' HDR' if self.is_video and self.prefer_hdr else '' + return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip() + + @property + def directory_path(self): + download_dir = Path(media_file_storage.location) + return download_dir / self.type_directory_path + + @property + def type_directory_path(self): + if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: + if self.is_audio: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + else: + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + else: + return Path(self.directory) + + def make_directory(self): + return os.makedirs(self.directory_path, exist_ok=True) + + @property + def get_image_url(self): + if self.is_playlist: + raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') + + return get_youtube_channel_image_info(self.url) + + + def directory_exists(self): + return (os.path.isdir(self.directory_path) and + os.access(self.directory_path, os.W_OK)) + + @property + def key_field(self): + return self.KEY_FIELD.get(self.source_type, '') + + @property + def source_resolution_height(self): + return SourceResolutionInteger.get(self.source_resolution, 0) + + @property + def can_fallback(self): + return self.fallback != Val(Fallback.FAIL) + + @property + def example_media_format_dict(self): + ''' + Populates a dict with real-ish and some placeholder data for media name + format strings. Used for example filenames and media_format validation. + ''' + fmt = [] + if self.source_resolution: + fmt.append(self.source_resolution) + if self.source_vcodec: + fmt.append(self.source_vcodec.lower()) + if self.source_acodec: + fmt.append(self.source_acodec.lower()) + if self.prefer_60fps: + fmt.append('60fps') + if self.prefer_hdr: + fmt.append('hdr') + now = timezone.now() + return { + 'yyyymmdd': now.strftime('%Y%m%d'), + 'yyyy_mm_dd': now.strftime('%Y-%m-%d'), + 'yyyy': now.strftime('%Y'), + 'mm': now.strftime('%m'), + 'dd': now.strftime('%d'), + 'source': self.slugname, + 'source_full': self.name, + 'uploader': 'Some Channel Name', + 'title': 'some-media-title-name', + 'title_full': 'Some Media Title Name', + 'key': 'SoMeUnIqUiD', + 'format': '-'.join(fmt), + 'playlist_title': 'Some Playlist Title', + 'video_order': '01', + 'ext': self.extension, + 'resolution': self.source_resolution if self.source_resolution else '', + 'height': '720' if self.source_resolution else '', + 'width': '1280' if self.source_resolution else '', + 'vcodec': self.source_vcodec.lower() if self.source_vcodec else '', + 'acodec': self.source_acodec.lower(), + 'fps': '24' if self.source_resolution else '', + 'hdr': 'hdr' if self.source_resolution else '' + } + + def get_example_media_format(self): + try: + return self.media_format.format(**self.example_media_format_dict) + except Exception as e: + return '' + + def is_regex_match(self, media_item_title): + if not self.filter_text: + return True + return bool(re.search(self.filter_text, media_item_title)) + + def get_index(self, type): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + days = None + if self.download_cap_date: + days = timezone.timedelta(seconds=self.download_cap).days + response = indexer(self.get_index_url(type=type), days=days) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + + def index_media(self): + ''' + Index the media source returning a list of media metadata as dicts. + ''' + entries = list() + if self.index_videos: + entries += self.get_index('videos') + # Playlists do something different that I have yet to figure out + if not self.is_playlist: + if self.index_streams: + entries += self.get_index('streams') + + if settings.MAX_ENTRIES_PROCESSING: + entries = entries[:settings.MAX_ENTRIES_PROCESSING] + return entries From 4b42670824f927c1a9d01661855e3e57c2cd8c74 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:24:02 -0400 Subject: [PATCH 338/454] Update __init__.py --- tubesync/sync/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index b144173f..ff0f7b3b 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,3 +1,4 @@ +from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer From b5ee002404e98a2ef31c97286e9e3c6aa0f401dd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:29:34 -0400 Subject: [PATCH 339/454] Update legacy.py --- tubesync/sync/models/legacy.py | 519 --------------------------------- 1 file changed, 519 deletions(-) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/legacy.py index a7facceb..83bd9256 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/legacy.py @@ -40,525 +40,6 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) -class Source(models.Model): - ''' - A Source is a source of media. Currently, this is either a YouTube channel - or a YouTube playlist. - ''' - - sponsorblock_categories = CommaSepChoiceField( - _(''), - max_length=128, - possible_choices=SponsorBlock_Category.choices, - all_choice='all', - allow_all=True, - all_label='(All Categories)', - default='all', - help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.') - ) - embed_metadata = models.BooleanField( - _('embed metadata'), - default=False, - help_text=_('Embed metadata from source into file') - ) - embed_thumbnail = models.BooleanField( - _('embed thumbnail'), - default=False, - help_text=_('Embed thumbnail into the file') - ) - enable_sponsorblock = models.BooleanField( - _('enable sponsorblock'), - default=True, - help_text=_('Use SponsorBlock?') - ) - - # Fontawesome icons used for the source on the front end - ICONS = _srctype_dict('') - - # Format to use to display a URL for the source - URLS = dict(zip( - YouTube_SourceType.values, - ( - 'https://www.youtube.com/c/{key}', - 'https://www.youtube.com/channel/{key}', - 'https://www.youtube.com/playlist?list={key}', - ), - )) - - # Format used to create indexable URLs - INDEX_URLS = dict(zip( - YouTube_SourceType.values, - ( - 'https://www.youtube.com/c/{key}/{type}', - 'https://www.youtube.com/channel/{key}/{type}', - 'https://www.youtube.com/playlist?list={key}', - ), - )) - - # Callback functions to get a list of media from the source - INDEXERS = _srctype_dict(get_youtube_media_info) - - # Field names to find the media ID used as the key when storing media - KEY_FIELD = _srctype_dict('id') - - uuid = models.UUIDField( - _('uuid'), - primary_key=True, - editable=False, - default=uuid.uuid4, - help_text=_('UUID of the source') - ) - created = models.DateTimeField( - _('created'), - auto_now_add=True, - db_index=True, - help_text=_('Date and time the source was created') - ) - last_crawl = models.DateTimeField( - _('last crawl'), - db_index=True, - null=True, - blank=True, - help_text=_('Date and time the source was last crawled') - ) - source_type = models.CharField( - _('source type'), - max_length=1, - db_index=True, - choices=YouTube_SourceType.choices, - default=YouTube_SourceType.CHANNEL, - help_text=_('Source type') - ) - key = models.CharField( - _('key'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Source key, such as exact YouTube channel name or playlist ID') - ) - name = models.CharField( - _('name'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Friendly name for the source, used locally in TubeSync only') - ) - directory = models.CharField( - _('directory'), - max_length=100, - db_index=True, - unique=True, - help_text=_('Directory name to save the media into') - ) - media_format = models.CharField( - _('media format'), - max_length=200, - default=settings.MEDIA_FORMATSTR_DEFAULT, - help_text=_('File format to use for saving files, detailed options at bottom of page.') - ) - index_schedule = models.IntegerField( - _('index schedule'), - choices=IndexSchedule.choices, - db_index=True, - default=IndexSchedule.EVERY_24_HOURS, - help_text=_('Schedule of how often to index the source for new media') - ) - download_media = models.BooleanField( - _('download media'), - default=True, - help_text=_('Download media from this source, if not selected the source will only be indexed') - ) - index_videos = models.BooleanField( - _('index videos'), - default=True, - help_text=_('Index video media from this source') - ) - index_streams = models.BooleanField( - _('index streams'), - default=False, - help_text=_('Index live stream media from this source') - ) - download_cap = models.IntegerField( - _('download cap'), - choices=CapChoices.choices, - default=CapChoices.CAP_NOCAP, - help_text=_('Do not download media older than this capped date') - ) - delete_old_media = models.BooleanField( - _('delete old media'), - default=False, - help_text=_('Delete old media after "days to keep" days?') - ) - days_to_keep = models.PositiveSmallIntegerField( - _('days to keep'), - default=14, - help_text=_('If "delete old media" is ticked, the number of days after which ' - 'to automatically delete media') - ) - filter_text = models.CharField( - _('filter string'), - max_length=200, - default='', - blank=True, - help_text=_('Regex compatible filter string for video titles') - ) - filter_text_invert = models.BooleanField( - _("invert filter text matching"), - default=False, - help_text="Invert filter string regex match, skip any matching titles when selected", - ) - filter_seconds = models.PositiveIntegerField( - _('filter seconds'), - blank=True, - null=True, - help_text=_('Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering') - ) - filter_seconds_min = models.BooleanField( - _('filter seconds min/max'), - choices=FilterSeconds.choices, - default=Val(FilterSeconds.MIN), - help_text=_('When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video ' - 'greater than maximum) video duration') - ) - delete_removed_media = models.BooleanField( - _('delete removed media'), - default=False, - help_text=_('Delete media that is no longer on this playlist') - ) - delete_files_on_disk = models.BooleanField( - _('delete files on disk'), - default=False, - help_text=_('Delete files on disk when they are removed from TubeSync') - ) - source_resolution = models.CharField( - _('source resolution'), - max_length=8, - db_index=True, - choices=SourceResolution.choices, - default=SourceResolution.VIDEO_1080P, - help_text=_('Source resolution, desired video resolution to download') - ) - source_vcodec = models.CharField( - _('source video codec'), - max_length=8, - db_index=True, - choices=list(reversed(YouTube_VideoCodec.choices)), - default=YouTube_VideoCodec.VP9, - help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)') - ) - source_acodec = models.CharField( - _('source audio codec'), - max_length=8, - db_index=True, - choices=list(reversed(YouTube_AudioCodec.choices)), - default=YouTube_AudioCodec.OPUS, - help_text=_('Source audio codec, desired audio encoding format to download') - ) - prefer_60fps = models.BooleanField( - _('prefer 60fps'), - default=True, - help_text=_('Where possible, prefer 60fps media for this source') - ) - prefer_hdr = models.BooleanField( - _('prefer hdr'), - default=False, - help_text=_('Where possible, prefer HDR media for this source') - ) - fallback = models.CharField( - _('fallback'), - max_length=1, - db_index=True, - choices=Fallback.choices, - default=Fallback.NEXT_BEST_HD, - help_text=_('What do do when media in your source resolution and codecs is not available') - ) - copy_channel_images = models.BooleanField( - _('copy channel images'), - default=False, - help_text=_('Copy channel banner and avatar. These may be detected and used by some media servers') - ) - copy_thumbnails = models.BooleanField( - _('copy thumbnails'), - default=False, - help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers') - ) - write_nfo = models.BooleanField( - _('write nfo'), - default=False, - help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers') - ) - write_json = models.BooleanField( - _('write json'), - default=False, - help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers') - ) - has_failed = models.BooleanField( - _('has failed'), - default=False, - help_text=_('Source has failed to index media') - ) - - write_subtitles = models.BooleanField( - _('write subtitles'), - default=False, - help_text=_('Download video subtitles') - ) - - auto_subtitles = models.BooleanField( - _('accept auto-generated subs'), - default=False, - help_text=_('Accept auto-generated subtitles') - ) - sub_langs = models.CharField( - _('subs langs'), - max_length=30, - default='en', - help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), - validators=[ - RegexValidator( - regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", - message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') - ) - ] - ) - - def __str__(self): - return self.name - - class Meta: - verbose_name = _('Source') - verbose_name_plural = _('Sources') - - @property - def icon(self): - return self.ICONS.get(self.source_type) - - @property - def slugname(self): - replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and') - return slugify(replaced)[:80] - - def deactivate(self): - self.download_media = False - self.index_streams = False - self.index_videos = False - self.index_schedule = IndexSchedule.NEVER - self.save(update_fields={ - 'download_media', - 'index_streams', - 'index_videos', - 'index_schedule', - }) - - @property - def is_active(self): - active = ( - self.download_media or - self.index_streams or - self.index_videos - ) - return self.index_schedule and active - - @property - def is_audio(self): - return self.source_resolution == SourceResolution.AUDIO.value - - @property - def is_playlist(self): - return self.source_type == YouTube_SourceType.PLAYLIST.value - - @property - def is_video(self): - return not self.is_audio - - @property - def download_cap_date(self): - delta = self.download_cap - if delta > 0: - return timezone.now() - timedelta(seconds=delta) - else: - return False - - @property - def days_to_keep_date(self): - delta = self.days_to_keep - if delta > 0: - return timezone.now() - timedelta(days=delta) - else: - return False - - @property - def extension(self): - ''' - The extension is also used by youtube-dl to set the output container. As - it is possible to quite easily pick combinations of codecs and containers - which are invalid (e.g. OPUS audio in an MP4 container) just set this for - people. All video is set to mkv containers, audio-only is set to m4a or ogg - depending on audio codec. - ''' - if self.is_audio: - if self.source_acodec == Val(YouTube_AudioCodec.MP4A): - return Val(FileExtension.M4A) - elif self.source_acodec == Val(YouTube_AudioCodec.OPUS): - return Val(FileExtension.OGG) - else: - raise ValueError('Unable to choose audio extension, uknown acodec') - else: - return Val(FileExtension.MKV) - - @classmethod - def create_url(obj, source_type, key): - url = obj.URLS.get(source_type) - return url.format(key=key) - - @classmethod - def create_index_url(obj, source_type, key, type): - url = obj.INDEX_URLS.get(source_type) - return url.format(key=key, type=type) - - @property - def url(self): - return Source.create_url(self.source_type, self.key) - - def get_index_url(self, type): - return Source.create_index_url(self.source_type, self.key, type) - - @property - def format_summary(self): - if self.is_audio: - vc = 'none' - else: - vc = self.source_vcodec - ac = self.source_acodec - f = ' 60FPS' if self.is_video and self.prefer_60fps else '' - h = ' HDR' if self.is_video and self.prefer_hdr else '' - return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip() - - @property - def directory_path(self): - download_dir = Path(media_file_storage.location) - return download_dir / self.type_directory_path - - @property - def type_directory_path(self): - if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: - if self.is_audio: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory - else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory - else: - return Path(self.directory) - - def make_directory(self): - return os.makedirs(self.directory_path, exist_ok=True) - - @property - def get_image_url(self): - if self.is_playlist: - raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') - - return get_youtube_channel_image_info(self.url) - - - def directory_exists(self): - return (os.path.isdir(self.directory_path) and - os.access(self.directory_path, os.W_OK)) - - @property - def key_field(self): - return self.KEY_FIELD.get(self.source_type, '') - - @property - def source_resolution_height(self): - return SourceResolutionInteger.get(self.source_resolution, 0) - - @property - def can_fallback(self): - return self.fallback != Val(Fallback.FAIL) - - @property - def example_media_format_dict(self): - ''' - Populates a dict with real-ish and some placeholder data for media name - format strings. Used for example filenames and media_format validation. - ''' - fmt = [] - if self.source_resolution: - fmt.append(self.source_resolution) - if self.source_vcodec: - fmt.append(self.source_vcodec.lower()) - if self.source_acodec: - fmt.append(self.source_acodec.lower()) - if self.prefer_60fps: - fmt.append('60fps') - if self.prefer_hdr: - fmt.append('hdr') - now = timezone.now() - return { - 'yyyymmdd': now.strftime('%Y%m%d'), - 'yyyy_mm_dd': now.strftime('%Y-%m-%d'), - 'yyyy': now.strftime('%Y'), - 'mm': now.strftime('%m'), - 'dd': now.strftime('%d'), - 'source': self.slugname, - 'source_full': self.name, - 'uploader': 'Some Channel Name', - 'title': 'some-media-title-name', - 'title_full': 'Some Media Title Name', - 'key': 'SoMeUnIqUiD', - 'format': '-'.join(fmt), - 'playlist_title': 'Some Playlist Title', - 'video_order': '01', - 'ext': self.extension, - 'resolution': self.source_resolution if self.source_resolution else '', - 'height': '720' if self.source_resolution else '', - 'width': '1280' if self.source_resolution else '', - 'vcodec': self.source_vcodec.lower() if self.source_vcodec else '', - 'acodec': self.source_acodec.lower(), - 'fps': '24' if self.source_resolution else '', - 'hdr': 'hdr' if self.source_resolution else '' - } - - def get_example_media_format(self): - try: - return self.media_format.format(**self.example_media_format_dict) - except Exception as e: - return '' - - def is_regex_match(self, media_item_title): - if not self.filter_text: - return True - return bool(re.search(self.filter_text, media_item_title)) - - def get_index(self, type): - indexer = self.INDEXERS.get(self.source_type, None) - if not callable(indexer): - raise Exception(f'Source type f"{self.source_type}" has no indexer') - days = None - if self.download_cap_date: - days = timedelta(seconds=self.download_cap).days - response = indexer(self.get_index_url(type=type), days=days) - if not isinstance(response, dict): - return [] - entries = response.get('entries', []) - return entries - - def index_media(self): - ''' - Index the media source returning a list of media metadata as dicts. - ''' - entries = list() - if self.index_videos: - entries += self.get_index('videos') - # Playlists do something different that I have yet to figure out - if not self.is_playlist: - if self.index_streams: - entries += self.get_index('streams') - - if settings.MAX_ENTRIES_PROCESSING: - entries = entries[:settings.MAX_ENTRIES_PROCESSING] - return entries - def get_media_thumb_path(instance, filename): # we don't want to use alternate names for thumb files if instance.thumb: From babc0345d6f9d31b2d27a95719501f62e73e0863 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:34:49 -0400 Subject: [PATCH 340/454] Update and rename legacy.py to media.py --- tubesync/sync/models/{legacy.py => media.py} | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) rename tubesync/sync/models/{legacy.py => media.py} (99%) diff --git a/tubesync/sync/models/legacy.py b/tubesync/sync/models/media.py similarity index 99% rename from tubesync/sync/models/legacy.py rename to tubesync/sync/models/media.py index 83bd9256..418e626d 100644 --- a/tubesync/sync/models/legacy.py +++ b/tubesync/sync/models/media.py @@ -36,9 +36,8 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, MediaState, SourceResolution, SourceResolutionInteger, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) - -media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') -_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) +from .source import Source +from .misc import media_file_storage, _srctype_dict def get_media_thumb_path(instance, filename): # we don't want to use alternate names for thumb files From e8777f2ecb4fd6c44d25b5501ed87a88559723f3 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:40:19 -0400 Subject: [PATCH 341/454] Create misc.py --- tubesync/sync/models/misc.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tubesync/sync/models/misc.py diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py new file mode 100644 index 00000000..383e0bba --- /dev/null +++ b/tubesync/sync/models/misc.py @@ -0,0 +1,8 @@ +from django.conf import settings +from django.core.files.storage import FileSystemStorage +from ..choices import Val, YouTube_SourceType + + +media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') +_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + From 37fcabb242976969846feb95c46e9316ff3c4816 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:42:20 -0400 Subject: [PATCH 342/454] Update __init__.py --- tubesync/sync/models/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index ff0f7b3b..f443c7c3 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,6 +1,5 @@ +from .media import Media from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer - -from .legacy import * From 6d288ec4477b906bdd46dd2f59c5e894b337b0cb Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:43:03 -0400 Subject: [PATCH 343/454] Update source.py --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 01a0d36e..ce0bf5ab 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -19,7 +19,7 @@ from ..youtube import ( get_media_info as get_youtube_media_info, get_channel_image_info as get_youtube_channel_image_info, ) -from .legacy import _srctype_dict, media_file_storage +from .misc import _srctype_dict, media_file_storage class Source(db.models.Model): From 7cad4ce396339d1587b0c4de3a6b99d3e587a595 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:45:42 -0400 Subject: [PATCH 344/454] Update source.py --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index ce0bf5ab..0dcfbc98 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -10,7 +10,7 @@ from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from ..choices import (Val, SponsorBlock_Category, YouTube_SourceType, IndexSchedule, - CapChoices, FilterSeconds, FileExtension, + CapChoices, Fallback, FileExtension, FilterSeconds, SourceResolution, SourceResolutionInteger, YouTube_VideoCodec, YouTube_AudioCodec, ) From 5b79293562dbdc3dfb9346a862700d54d3e24217 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:47:58 -0400 Subject: [PATCH 345/454] Update source.py --- tubesync/sync/models/source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 0dcfbc98..7ab9ea4b 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -5,6 +5,7 @@ from pathlib import Path from django import db from django.conf import settings from django.core.exceptions import SuspiciousOperation +from django.core.validators import RegexValidator from django.utils import timezone from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ From 109608c9af94c33339e85cc5965cbd25b097172d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:48:48 -0400 Subject: [PATCH 346/454] Update metadata.py --- tubesync/sync/models/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index e062ea0c..17d214fb 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -4,7 +4,7 @@ from common.timestamp import timestamp_to_datetime from common.utils import django_queryset_generator as qs_gen from django import db from django.utils.translation import gettext_lazy as _ -from .legacy import Media +from .media import Media class Metadata(db.models.Model): From 4cd2178edb55b6f2bb63679b24f816180a37cc40 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:51:30 -0400 Subject: [PATCH 347/454] Update __init__.py --- tubesync/sync/models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index f443c7c3..a9324f0e 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -3,3 +3,5 @@ from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer + +from .misc import media_file_storage From 004d98efa54e5f5149036a3647bec0a731f86d6b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 01:58:28 -0400 Subject: [PATCH 348/454] Update media.py --- tubesync/sync/models/media.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 418e626d..7f179f13 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -37,20 +37,10 @@ from ..choices import ( Val, CapChoices, Fallback, FileExtension, SponsorBlock_Category, YouTube_AudioCodec, YouTube_SourceType, YouTube_VideoCodec) from .source import Source -from .misc import media_file_storage, _srctype_dict - -def get_media_thumb_path(instance, filename): - # we don't want to use alternate names for thumb files - if instance.thumb: - instance.thumb.delete(save=False) - fileid = str(instance.uuid).lower() - filename = f'{fileid}.jpg' - prefix = fileid[:2] - return Path('thumbs') / prefix / filename - - -def get_media_file_path(instance, filename): - return instance.filepath +from .misc import ( + media_file_storage, _srctype_dict, + get_media_thumb_path, get_media_file_path, +) class Media(models.Model): From 951b24bbefddd8d31b8c90e90ce99b56f8a4da98 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:01:49 -0400 Subject: [PATCH 349/454] Update misc.py --- tubesync/sync/models/misc.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py index 383e0bba..159e0fa3 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/misc.py @@ -1,3 +1,4 @@ +from pathlib import Path from django.conf import settings from django.core.files.storage import FileSystemStorage from ..choices import Val, YouTube_SourceType @@ -6,3 +7,17 @@ from ..choices import Val, YouTube_SourceType media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + +def get_media_file_path(instance, filename): + return instance.filepath + + +def get_media_thumb_path(instance, filename): + # we don't want to use alternate names for thumb files + if instance.thumb: + instance.thumb.delete(save=False) + fileid = str(instance.uuid).lower() + filename = f'{fileid}.jpg' + prefix = fileid[:2] + return Path('thumbs') / prefix / filename + From 35697177b78651d357d10551526ce4e2c31df37d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:03:05 -0400 Subject: [PATCH 350/454] Update __init__.py --- tubesync/sync/models/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index a9324f0e..e28c7db7 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -4,4 +4,8 @@ from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer -from .misc import media_file_storage +from .misc import ( + get_media_file_path, + get_media_thumb_path, + media_file_storage, +) From 971be3d3dd7c584cb1d03b32adddc287dee604d9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:05:53 -0400 Subject: [PATCH 351/454] Update __init__.py --- tubesync/sync/models/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index e28c7db7..124e1ad9 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,3 +1,5 @@ +from common.json import JSONEncoder + from .media import Media from .source import Source from .metadata import Metadata @@ -5,7 +7,7 @@ from .metadata_format import MetadataFormat from .media_server import MediaServer from .misc import ( - get_media_file_path, + #get_media_file_path, get_media_thumb_path, media_file_storage, ) From 46e6030d114b4798884b5f9a42c44379631128ec Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:07:21 -0400 Subject: [PATCH 352/454] Update __init__.py --- tubesync/sync/models/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 124e1ad9..cd7861d3 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -7,7 +7,7 @@ from .metadata_format import MetadataFormat from .media_server import MediaServer from .misc import ( - #get_media_file_path, - get_media_thumb_path, + get_media_file_path, + #get_media_thumb_path, media_file_storage, ) From f9d9e7b66bcc750f70933e0be543ac69a08d7fab Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:08:47 -0400 Subject: [PATCH 353/454] Update __init__.py --- tubesync/sync/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index cd7861d3..f2649d4d 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -8,6 +8,6 @@ from .media_server import MediaServer from .misc import ( get_media_file_path, - #get_media_thumb_path, + get_media_thumb_path, media_file_storage, ) From 080acb7995840b5b1030d36f6fdf5a865222b1d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:10:52 -0400 Subject: [PATCH 354/454] Update __init__.py --- tubesync/sync/models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index f2649d4d..e951340d 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,5 +1,7 @@ from common.json import JSONEncoder +from ..fields import CommaSepChoiceField + from .media import Media from .source import Source from .metadata import Metadata From 0a59c1dd97898813d820d6846dafc4db2a749cb1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:24:14 -0400 Subject: [PATCH 355/454] Update media.py --- tubesync/sync/models/media.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 7f179f13..6be36b83 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1,7 +1,6 @@ import os import uuid import json -import re from collections import OrderedDict from copy import deepcopy from datetime import datetime, timedelta, timezone as tz @@ -9,16 +8,12 @@ from pathlib import Path from xml.etree import ElementTree from django.conf import settings from django.db import models -from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation -from django.core.files.storage import FileSystemStorage -from django.core.serializers.json import DjangoJSONEncoder -from django.core.validators import RegexValidator +from django.core.exceptions import ObjectDoesNotExist from django.db.transaction import atomic from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log -from common.json import JSONEncoder from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) @@ -30,7 +25,6 @@ from ..utils import (seconds_to_timestr, parse_media_format, filter_response, multi_key_sort) from ..matching import ( get_best_combined_format, get_best_audio_format, get_best_video_format) -from ..fields import CommaSepChoiceField from ..choices import ( Val, CapChoices, Fallback, FileExtension, FilterSeconds, IndexSchedule, MediaServerType, MediaState, SourceResolution, SourceResolutionInteger, From 1a3a86cde422840aeedfb8da1f16dc8c90c5902a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 02:37:04 -0400 Subject: [PATCH 356/454] Update settings.py --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 9960f60e..391a209d 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.0' +VERSION = '0.15.1' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From 6278087b12ef1f29f4c440a30db6be3da751f42a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 03:04:12 -0400 Subject: [PATCH 357/454] Update media.py --- tubesync/sync/models/media.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 6be36b83..ce351267 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,19 +17,15 @@ from common.logger import log from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, django_queryset_generator as qs_gen, ) -from ..youtube import ( get_media_info as get_youtube_media_info, - download_media as download_youtube_media, - get_channel_image_info as get_youtube_channel_image_info) +from ..youtube import ( get_media_info as get_youtube_media_info, + download_media as download_youtube_media) from ..utils import (seconds_to_timestr, parse_media_format, filter_response, write_text_file, mkdir_p, directory_and_stem, glob_quote, multi_key_sort) -from ..matching import ( get_best_combined_format, get_best_audio_format, +from ..matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) -from ..choices import ( Val, CapChoices, Fallback, FileExtension, - FilterSeconds, IndexSchedule, MediaServerType, - MediaState, SourceResolution, SourceResolutionInteger, - SponsorBlock_Category, YouTube_AudioCodec, - YouTube_SourceType, YouTube_VideoCodec) +from ..choices import ( Val, Fallback, MediaState, SourceResolution, + YouTube_AudioCodec, YouTube_VideoCodec) from .source import Source from .misc import ( media_file_storage, _srctype_dict, From 3bb19861cd9295dbb7bca7cd0748d4c8a22a9f86 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 03:40:20 -0400 Subject: [PATCH 358/454] Add `_nfo_element` function --- tubesync/sync/models/misc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/misc.py index 159e0fa3..81257966 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/misc.py @@ -8,6 +8,13 @@ media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), bas _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) +def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): + element = nfo.makeelement(label, attrs) + element.text = text + element.tail = tail + (char * indent) + return element + + def get_media_file_path(instance, filename): return instance.filepath From 189df4d72ae2f4600970c64facd329be8d2615df Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 04:17:48 -0400 Subject: [PATCH 359/454] Use `_nfo_element` function --- tubesync/sync/models/media.py | 116 ++++++++++++++-------------------- 1 file changed, 48 insertions(+), 68 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index ce351267..eed6103a 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -28,7 +28,7 @@ from ..choices import ( Val, Fallback, MediaState, SourceResolution, YouTube_AudioCodec, YouTube_VideoCodec) from .source import Source from .misc import ( - media_file_storage, _srctype_dict, + media_file_storage, _srctype_dict, _nfo_element, get_media_thumb_path, get_media_file_path, ) @@ -942,37 +942,27 @@ class Media(models.Model): nfo = ElementTree.Element('episodedetails') nfo.text = '\n ' # title = media metadata title - title = nfo.makeelement('title', {}) - title.text = clean_emoji(self.title) - title.tail = '\n ' - nfo.append(title) + nfo.append(_nfo_element(nfo, + 'title', clean_emoji(self.title), + )) # showtitle = source name - showtitle = nfo.makeelement('showtitle', {}) - showtitle.text = clean_emoji(str(self.source.name).strip()) - showtitle.tail = '\n ' - nfo.append(showtitle) + nfo.append(_nfo_element(nfo, + 'showtitle', clean_emoji(str(self.source.name).strip()), + )) # season = upload date year - season = nfo.makeelement('season', {}) - if self.source.is_playlist: - # If it's a playlist, set season to 1 - season.text = '1' - else: - # If it's not a playlist, set season to upload date year - season.text = str(self.upload_date.year) if self.upload_date else '' - season.tail = '\n ' - nfo.append(season) + nfo.append(_nfo_element(nfo, + 'season', + '1' if self.source.is_playlist else str( + self.upload_date.year if self.upload_date else '' + ), + )) # episode = number of video in the year - episode = nfo.makeelement('episode', {}) - episode.text = self.get_episode_str() - episode.tail = '\n ' - nfo.append(episode) + nfo.append(_nfo_element(nfo, + 'episode', self.get_episode_str(), + )) # ratings = media metadata youtube rating - value = nfo.makeelement('value', {}) - value.text = str(self.rating) - value.tail = '\n ' - votes = nfo.makeelement('votes', {}) - votes.text = str(self.votes) - votes.tail = '\n ' + value = _nfo_element(nfo, 'value', str(self.rating), indent=6) + votes = _nfo_element(nfo, 'votes', str(self.votes), indent=4) rating_attrs = OrderedDict() rating_attrs['name'] = 'youtube' rating_attrs['max'] = '5' @@ -989,61 +979,51 @@ class Media(models.Model): ratings.tail = '\n ' nfo.append(ratings) # plot = media metadata description - plot = nfo.makeelement('plot', {}) - plot.text = clean_emoji(str(self.description).strip()) - plot.tail = '\n ' - nfo.append(plot) + nfo.append(_nfo_element(nfo, + 'plot', clean_emoji(str(self.description).strip()), + )) # thumb = local path to media thumbnail - thumb = nfo.makeelement('thumb', {}) - thumb.text = self.thumbname if self.source.copy_thumbnails else '' - thumb.tail = '\n ' - nfo.append(thumb) + nfo.append(_nfo_element(nfo, + 'thumb', self.thumbname if self.source.copy_thumbnails else '', + )) # mpaa = media metadata age requirement - mpaa = nfo.makeelement('mpaa', {}) - mpaa.text = str(self.age_limit) - mpaa.tail = '\n ' if self.age_limit and self.age_limit > 0: - nfo.append(mpaa) + nfo.append(_nfo_element(nfo, + 'mpaa', str(self.age_limit), + )) # runtime = media metadata duration in seconds - runtime = nfo.makeelement('runtime', {}) - runtime.text = str(self.duration) - runtime.tail = '\n ' - nfo.append(runtime) + nfo.append(_nfo_element(nfo, + 'runtime', str(self.duration), + )) # id = media key - idn = nfo.makeelement('id', {}) - idn.text = str(self.key).strip() - idn.tail = '\n ' - nfo.append(idn) + nfo.append(_nfo_element(nfo, + 'id', str(self.key).strip(), + )) # uniqueid = media key uniqueid_attrs = OrderedDict() uniqueid_attrs['type'] = 'youtube' uniqueid_attrs['default'] = 'True' - uniqueid = nfo.makeelement('uniqueid', uniqueid_attrs) - uniqueid.text = str(self.key).strip() - uniqueid.tail = '\n ' - nfo.append(uniqueid) + nfo.append(_nfo_element(nfo, + 'uniqueid', str(self.key).strip(), attrs=uniqueid_attrs, + )) # studio = media metadata uploader - studio = nfo.makeelement('studio', {}) - studio.text = clean_emoji(str(self.uploader).strip()) - studio.tail = '\n ' - nfo.append(studio) + nfo.append(_nfo_element(nfo, + 'studio', clean_emoji(str(self.uploader).strip()), + )) # aired = media metadata uploaded date - aired = nfo.makeelement('aired', {}) upload_date = self.upload_date - aired.text = upload_date.strftime('%Y-%m-%d') if upload_date else '' - aired.tail = '\n ' - nfo.append(aired) + nfo.append(_nfo_element(nfo, + 'aired', upload_date.strftime('%Y-%m-%d') if upload_date else '', + )) # dateadded = date and time media was created in tubesync - dateadded = nfo.makeelement('dateadded', {}) - dateadded.text = self.created.strftime('%Y-%m-%d %H:%M:%S') - dateadded.tail = '\n ' - nfo.append(dateadded) + nfo.append(_nfo_element(nfo, + 'dateadded', self.created.strftime('%Y-%m-%d %H:%M:%S'), + )) # genre = any media metadata categories if they exist for category_str in self.categories: - genre = nfo.makeelement('genre', {}) - genre.text = str(category_str).strip() - genre.tail = '\n ' - nfo.append(genre) + nfo.append(_nfo_element(nfo, + 'genre', str(category_str).strip(), + )) nfo[-1].tail = '\n' # Return XML tree as a prettified string return ElementTree.tostring(nfo, encoding='utf8', method='xml').decode('utf8') From 9aa5e6b9675f4b0d9c501842baa010237927814b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:54:43 -0400 Subject: [PATCH 360/454] Create a media entry after deletion --- tubesync/sync/signals.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 32b0b5f6..8218c438 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,6 +295,25 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=instance.get_metadata_first_value( + 'thumbnail', + arg_dict=existing_metadata, + ), + ), + ) + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -370,3 +389,32 @@ def media_post_delete(sender, instance, **kwargs): log.info(f'Deleting file for: {instance} path: {file}') delete_file(file) + # Create a media entry for the indexing task to find + # Requirements: + # source, key, duration, title, published + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) + if created: + old_metadata = instance.loaded_metadata + skipped_media.downloaded = False + skipped_media.duration = instance.duration + skipped_media.metadata = skipped_media.metadata_dumps( + arg_dict=dict( + _media_instance_was_deleted=True, + site=old_metadata.get('site'), + thumbnail=old_metadata.get('thumbnail'), + ), + ) + skipped_media.published = instance.published + skipped_media.title = instance.title + skipped_media.skip = True + skipped_media.manual_skip = True + skipped_media.save() + Metadata.objects.filter( + media__isnull=True, + site=old_metadata.get('site') or 'Youtube', + key=skipped_media.key, + ).update(media=skipped_media) + From f86b666c125446a116c1fddae5ecc90463bc3f52 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 14:57:23 -0400 Subject: [PATCH 361/454] fixup: import `Metadata` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 8218c438..49b0145d 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from .models import Source, Media, MediaServer +from .models import Source, Media, MediaServer, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, From d1fd568066a27c5804a097b34d47e57de3a141d6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:11:18 -0400 Subject: [PATCH 362/454] Test for the specific media items, not any task --- tubesync/sync/tests.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 303aa18a..b5440291 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,8 +469,19 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} - download_media_thumbnail_tasks = Task.objects.filter(**q) - self.assertFalse(download_media_thumbnail_tasks) + found_thumbnail_task1 = False + found_thumbnail_task2 = False + found_thumbnail_task3 = False + for task in Task.objects.filter(**q): + if test_media1_pk in task.task_params: + found_thumbnail_task1 = True + if test_media2_pk in task.task_params: + found_thumbnail_task2 = True + if test_media3_pk in task.task_params: + found_thumbnail_task3 = True + self.assertFalse(found_thumbnail_task1) + self.assertFalse(found_thumbnail_task2) + self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 4a04326d16737f9bbc7511ddc3c6c0e8f2129c92 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:31:49 -0400 Subject: [PATCH 363/454] Change the expected count and verify the key matches --- tubesync/sync/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b5440291..b84ffe03 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1847,5 +1847,6 @@ class TasksTestCase(TestCase): cleanup_old_media() self.assertEqual(src1.media_source.all().count(), 3) - self.assertEqual(src2.media_source.all().count(), 2) + self.assertEqual(src2.media_source.all().count(), 3) self.assertEqual(Media.objects.filter(pk=m22.pk).exists(), False) + self.assertEqual(Media.objects.filter(source=src2, key=m22.key, skip=True).exists(), True) From ba451d9401a0834a80b6d92f578e88497b704352 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:34:53 -0400 Subject: [PATCH 364/454] testing: sanity check --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 49b0145d..877cc4ed 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -396,7 +396,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if created: + if False and created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From a80623bce40e51a1755e4af448fb944d331cf964 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:41:14 -0400 Subject: [PATCH 365/454] testing: no save in pre-delete --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 877cc4ed..efb273a0 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,7 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - instance.save() + # TODO: instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) From 5a9c3050c8069d07092964af27a7682970b5dcfd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 15:50:37 -0400 Subject: [PATCH 366/454] Fixes from CI tests --- tubesync/sync/signals.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index efb273a0..26b18ecc 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,7 +313,9 @@ def media_pre_delete(sender, instance, **kwargs): ), ), ) - # TODO: instance.save() + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -396,7 +398,7 @@ def media_post_delete(sender, instance, **kwargs): key=instance.key, source=instance.source, ) - if False and created: + if created: old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration From 9d8a5574e416b0a9881e4777c44d8325e46ccb29 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:01:15 -0400 Subject: [PATCH 367/454] Delete tasks first then modify metadata --- tubesync/sync/signals.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 26b18ecc..76731598 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -295,27 +295,6 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): - # Save the metadata site & thumbnail URL to the metadata column - existing_metadata = instance.loaded_metadata - metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( - deleted=True, - site=instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail=instance.get_metadata_first_value( - 'thumbnail', - arg_dict=existing_metadata, - ), - ), - ) - # Do not create more tasks before deleting - instance.manual_skip = True - instance.save() # Triggered before media is deleted, delete any unlocked scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) @@ -331,6 +310,24 @@ def media_pre_delete(sender, instance, **kwargs): # Remove thumbnail file for deleted media if instance.thumb: instance.thumb.delete(save=False) + # Save the metadata site & thumbnail URL to the metadata column + existing_metadata = instance.loaded_metadata + metadata_str = instance.metadata or '{}' + column_metadata = instance.metadata_loads(metadata_str) + instance.metadata = instance.metadata_dumps( + arg_dict=dict(column_metadata).update( + deleted=True, + site=instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail=thumbnail_url, + ), + ) + # Do not create more tasks before deleting + instance.manual_skip = True + instance.save() @receiver(post_delete, sender=Media) From f7ca8189a0e61cf8e3f6b1f6c065c2f45d7fa5f1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:04:27 -0400 Subject: [PATCH 368/454] Update tests.py --- tubesync/sync/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index b84ffe03..2926a296 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -469,10 +469,12 @@ class FrontEndTestCase(TestCase): self.assertEqual(response.status_code, 404) # Confirm any tasks have been deleted q = {'task_name': 'sync.tasks.download_media_thumbnail'} + download_media_thumbnail_tasks = Task.objects.filter(**q) + self.assertFalse(download_media_thumbnail_tasks) found_thumbnail_task1 = False found_thumbnail_task2 = False found_thumbnail_task3 = False - for task in Task.objects.filter(**q): + for task in download_media_thumbnail_tasks: if test_media1_pk in task.task_params: found_thumbnail_task1 = True if test_media2_pk in task.task_params: From 147f245e97d1cabd31955496679fedc87324ad1f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 16:06:28 -0400 Subject: [PATCH 369/454] Update tests.py --- tubesync/sync/tests.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 2926a296..24f0d092 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -471,19 +471,6 @@ class FrontEndTestCase(TestCase): q = {'task_name': 'sync.tasks.download_media_thumbnail'} download_media_thumbnail_tasks = Task.objects.filter(**q) self.assertFalse(download_media_thumbnail_tasks) - found_thumbnail_task1 = False - found_thumbnail_task2 = False - found_thumbnail_task3 = False - for task in download_media_thumbnail_tasks: - if test_media1_pk in task.task_params: - found_thumbnail_task1 = True - if test_media2_pk in task.task_params: - found_thumbnail_task2 = True - if test_media3_pk in task.task_params: - found_thumbnail_task3 = True - self.assertFalse(found_thumbnail_task1) - self.assertFalse(found_thumbnail_task2) - self.assertFalse(found_thumbnail_task3) q = {'task_name': 'sync.tasks.download_media'} download_media_tasks = Task.objects.filter(**q) self.assertFalse(download_media_tasks) From 7a8421e11acf0921eee516d13e7395934b6b1f01 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:10:14 -0400 Subject: [PATCH 370/454] Use the field mapping for site & thumbnail --- tubesync/sync/signals.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 76731598..9bec1401 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -314,16 +314,19 @@ def media_pre_delete(sender, instance, **kwargs): existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' column_metadata = instance.metadata_loads(metadata_str) + site_field = instance.get_metadata_field('extractor_key') + thumbnail_field = instance.get_metadata_field('thumbnail') instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update( + arg_dict=dict(column_metadata).update(dict( deleted=True, - site=instance.get_metadata_first_value( + ).update({ + site_field: instance.get_metadata_first_value( 'extractor_key', 'Youtube', arg_dict=existing_metadata, ), - thumbnail=thumbnail_url, - ), + thumbnail_field: thumbnail_url, + })), ) # Do not create more tasks before deleting instance.manual_skip = True @@ -396,15 +399,19 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + site_field = instance.get_metadata_field('extractor_key') + thumbnail_url = instance.thumbnail + thumbnail_field = instance.get_metadata_field('thumbnail') old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration skipped_media.metadata = skipped_media.metadata_dumps( arg_dict=dict( _media_instance_was_deleted=True, - site=old_metadata.get('site'), - thumbnail=old_metadata.get('thumbnail'), - ), + ).update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }), ) skipped_media.published = instance.published skipped_media.title = instance.title @@ -413,7 +420,7 @@ def media_post_delete(sender, instance, **kwargs): skipped_media.save() Metadata.objects.filter( media__isnull=True, - site=old_metadata.get('site') or 'Youtube', + site=old_metadata.get(site_field) or 'Youtube', key=skipped_media.key, ).update(media=skipped_media) From 691d049e2cd74f36882c4bf9ae759f9f0d367e3b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 17:39:57 -0400 Subject: [PATCH 371/454] fixup: update `dict` first --- tubesync/sync/signals.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 9bec1401..1743914a 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -313,21 +313,18 @@ def media_pre_delete(sender, instance, **kwargs): # Save the metadata site & thumbnail URL to the metadata column existing_metadata = instance.loaded_metadata metadata_str = instance.metadata or '{}' - column_metadata = instance.metadata_loads(metadata_str) + arg_dict = instance.metadata_loads(metadata_str) site_field = instance.get_metadata_field('extractor_key') thumbnail_field = instance.get_metadata_field('thumbnail') - instance.metadata = instance.metadata_dumps( - arg_dict=dict(column_metadata).update(dict( - deleted=True, - ).update({ - site_field: instance.get_metadata_first_value( - 'extractor_key', - 'Youtube', - arg_dict=existing_metadata, - ), - thumbnail_field: thumbnail_url, - })), - ) + arg_dict.update({ + site_field: instance.get_metadata_first_value( + 'extractor_key', + 'Youtube', + arg_dict=existing_metadata, + ), + thumbnail_field: thumbnail_url, + }) + instance.metadata = instance.metadata_dumps(arg_dict=arg_dict) # Do not create more tasks before deleting instance.manual_skip = True instance.save() @@ -399,19 +396,21 @@ def media_post_delete(sender, instance, **kwargs): source=instance.source, ) if created: + old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key') thumbnail_url = instance.thumbnail thumbnail_field = instance.get_metadata_field('thumbnail') - old_metadata = instance.loaded_metadata skipped_media.downloaded = False skipped_media.duration = instance.duration + arg_dict=dict( + _media_instance_was_deleted=True, + ) + arg_dict.update({ + site_field: old_metadata.get(site_field), + thumbnail_field: thumbnail_url, + }) skipped_media.metadata = skipped_media.metadata_dumps( - arg_dict=dict( - _media_instance_was_deleted=True, - ).update({ - site_field: old_metadata.get(site_field), - thumbnail_field: thumbnail_url, - }), + arg_dict=arg_dict, ) skipped_media.published = instance.published skipped_media.title = instance.title From 7beda36d87523c802c1b076925c85f6813e1d5a2 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:02:03 -0400 Subject: [PATCH 372/454] Mark media for deletion --- tubesync/sync/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2e6b1433..d29e8239 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -933,6 +933,10 @@ def delete_all_media_for_source(source_id, source_name, source_directory): for media in qs_gen(mqs): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): + #media.downloaded = False + media.skip = True + media.manual_skip = True + media.save() media.delete() # Remove the directory, if the user requested that directory_path = Path(source_directory) From 071508e2df128cae07ff3d0e922530627da7a4cd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 9 May 2025 22:20:57 -0400 Subject: [PATCH 373/454] Coordination with deletion of `Source` instances --- tubesync/sync/signals.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 1743914a..6a2dd22f 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -333,7 +333,13 @@ def media_pre_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): # Remove the video file, when configured to do so - if instance.source.delete_files_on_disk and instance.media_file: + remove_files = ( + instance.source and + instance.source.delete_files_on_disk and + instance.downloaded and + instance.media_file + ) + if remove_files: video_path = Path(str(instance.media_file.path)).resolve(strict=False) instance.media_file.delete(save=False) # the other files we created have these known suffixes @@ -391,10 +397,19 @@ def media_post_delete(sender, instance, **kwargs): # Create a media entry for the indexing task to find # Requirements: # source, key, duration, title, published - skipped_media, created = Media.objects.get_or_create( - key=instance.key, - source=instance.source, + created = False + create_for_indexing_task = ( + not ( + #not instance.downloaded and + instance.skip and + instance.manual_skip + ) ) + if create_for_indexing_task: + skipped_media, created = Media.objects.get_or_create( + key=instance.key, + source=instance.source, + ) if created: old_metadata = instance.loaded_metadata site_field = instance.get_metadata_field('extractor_key') From 2a075720e1e51c94ba4da135336af8ca8dbef268 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 01:29:29 -0400 Subject: [PATCH 374/454] Re-order imports --- tubesync/sync/models/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index e951340d..362095e7 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -2,14 +2,15 @@ from common.json import JSONEncoder from ..fields import CommaSepChoiceField +from .misc import ( + get_media_file_path, + get_media_thumb_path, + media_file_storage, +) + from .media import Media from .source import Source from .metadata import Metadata from .metadata_format import MetadataFormat from .media_server import MediaServer -from .misc import ( - get_media_file_path, - get_media_thumb_path, - media_file_storage, -) From e697fc70a11a39622dd5c6e7abc1c4ccb00f0a21 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 01:36:16 -0400 Subject: [PATCH 375/454] Use `JSONEncoder` for `MediaServer` --- tubesync/sync/models/media_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/media_server.py b/tubesync/sync/models/media_server.py index a8ecefc6..74502fac 100644 --- a/tubesync/sync/models/media_server.py +++ b/tubesync/sync/models/media_server.py @@ -1,3 +1,4 @@ +from common.json import JSONEncoder from django import db from django.utils.translation import gettext_lazy as _ from ..choices import Val, MediaServerType @@ -45,6 +46,7 @@ class MediaServer(db.models.Model): ) options = db.models.JSONField( _('options'), + encoder=JSONEncoder, blank=False, null=True, help_text=_('Options for the media server'), From 25f5c1cf5b2c9ea4a81de254411b80b0fae7855e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 02:26:02 -0400 Subject: [PATCH 376/454] Add migration file via upload --- ...ons_alter_source_source_acodec_and_more.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py diff --git a/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py b/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py new file mode 100644 index 00000000..46ea113f --- /dev/null +++ b/tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.1.9 on 2025-05-10 06:18 + +import common.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0032_metadata_transfer'), + ] + + operations = [ + migrations.AlterField( + model_name='mediaserver', + name='options', + field=models.JSONField(encoder=common.json.JSONEncoder, help_text='Options for the media server', null=True, verbose_name='options'), + ), + migrations.AlterField( + model_name='source', + name='source_acodec', + field=models.CharField(choices=[('OPUS', 'OPUS'), ('MP4A', 'MP4A')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'), + ), + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AV1', 'AV1'), ('VP9', 'VP9'), ('AVC1', 'AVC1 (H.264)')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + ] From a927c0f6de8d91a56594f2efb2397e16d139973e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:08:58 -0400 Subject: [PATCH 377/454] Add comments about the imports --- tubesync/sync/models/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 362095e7..41028293 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,13 +1,21 @@ +# These are referenced from the migration files +# TODO: update migration files to remove +# CommaSepChoiceField and JSONEncoder + from common.json import JSONEncoder from ..fields import CommaSepChoiceField +# Used by migration files and staying here + from .misc import ( get_media_file_path, get_media_thumb_path, media_file_storage, ) +# The actual model classes + from .media import Media from .source import Source from .metadata import Metadata From 58c20d669a4c85cf91e07aac1d37e0a960ab32fc Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:17:15 -0400 Subject: [PATCH 378/454] Update 0031_metadata_metadataformat.py --- tubesync/sync/migrations/0031_metadata_metadataformat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/migrations/0031_metadata_metadataformat.py b/tubesync/sync/migrations/0031_metadata_metadataformat.py index 00efa0f6..aee89518 100644 --- a/tubesync/sync/migrations/0031_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_metadata_metadataformat.py @@ -1,7 +1,7 @@ # Generated by Django 5.1.8 on 2025-04-11 07:36 +import common.json import django.db.models.deletion -import sync.models import uuid from django.db import migrations, models @@ -23,7 +23,7 @@ class Migration(migrations.Migration): ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')), ], options={ @@ -40,7 +40,7 @@ class Migration(migrations.Migration): ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')), ], options={ From 21974612999a0a7eabc4f706e206bb1a8664098d Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:20:13 -0400 Subject: [PATCH 379/454] Update 0031_squashed_metadata_metadataformat.py --- .../migrations/0031_squashed_metadata_metadataformat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py index 13189f10..c7a78bd8 100644 --- a/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py +++ b/tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py @@ -1,7 +1,7 @@ # Generated by Django 5.1.8 on 2025-04-23 18:10 +import common.json import django.db.models.deletion -import sync.models import uuid from django.db import migrations, models @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')), ('uploaded', models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')), ('published', models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')), ('media', models.OneToOneField(help_text='Media the metadata belongs to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='new_metadata', to='sync.media')), ], options={ @@ -43,7 +43,7 @@ class Migration(migrations.Migration): ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key')), ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')), ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')), - ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), + ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')), ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata')), ], options={ From 24d126ad7d47e4931eba89800d77c4d8ff53bedd Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:22:05 -0400 Subject: [PATCH 380/454] Update __init__.py --- tubesync/sync/models/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 41028293..3e4d521b 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,8 +1,6 @@ # These are referenced from the migration files # TODO: update migration files to remove -# CommaSepChoiceField and JSONEncoder - -from common.json import JSONEncoder +# CommaSepChoiceField from ..fields import CommaSepChoiceField From e1faca4c3d8af84d205326179aad5d5fa0eba953 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:29:59 -0400 Subject: [PATCH 381/454] Update 0016_auto_20230214_2052.py --- tubesync/sync/migrations/0016_auto_20230214_2052.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/migrations/0016_auto_20230214_2052.py b/tubesync/sync/migrations/0016_auto_20230214_2052.py index ffba1952..d4319759 100644 --- a/tubesync/sync/migrations/0016_auto_20230214_2052.py +++ b/tubesync/sync/migrations/0016_auto_20230214_2052.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.18 on 2023-02-14 20:52 from django.db import migrations, models -import sync.models +import sync.fields class Migration(migrations.Migration): @@ -29,6 +29,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='source', name='sponsorblock_categories', - field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), + field=sync.fields.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), ), ] From 26372efe25b52aaf666d60c466c19c4b98d73bf6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 03:31:52 -0400 Subject: [PATCH 382/454] Update __init__.py --- tubesync/sync/models/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index 3e4d521b..dba06dfb 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,10 +1,4 @@ # These are referenced from the migration files -# TODO: update migration files to remove -# CommaSepChoiceField - -from ..fields import CommaSepChoiceField - -# Used by migration files and staying here from .misc import ( get_media_file_path, From 0d1f066fced3d1063b9fe56e2862188b4396a276 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 04:21:06 -0400 Subject: [PATCH 383/454] Reorganize misc.py --- tubesync/sync/models/__init__.py | 13 ++++++++----- tubesync/sync/models/{misc.py => _migrations.py} | 9 --------- tubesync/sync/models/_private.py | 12 ++++++++++++ tubesync/sync/models/media.py | 8 ++++---- tubesync/sync/models/source.py | 3 ++- 5 files changed, 26 insertions(+), 19 deletions(-) rename tubesync/sync/models/{misc.py => _migrations.py} (62%) create mode 100644 tubesync/sync/models/_private.py diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index dba06dfb..d7ed077c 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -1,16 +1,19 @@ # These are referenced from the migration files -from .misc import ( +from ._migrations import ( get_media_file_path, get_media_thumb_path, media_file_storage, ) # The actual model classes +# The order starts with independent classes +# then the classes that depend on them follow. -from .media import Media -from .source import Source -from .metadata import Metadata -from .metadata_format import MetadataFormat from .media_server import MediaServer +from .source import Source +from .media import Media +from .metadata import Metadata +from .metadata_format import MetadataFormat + diff --git a/tubesync/sync/models/misc.py b/tubesync/sync/models/_migrations.py similarity index 62% rename from tubesync/sync/models/misc.py rename to tubesync/sync/models/_migrations.py index 81257966..5ca5d101 100644 --- a/tubesync/sync/models/misc.py +++ b/tubesync/sync/models/_migrations.py @@ -1,18 +1,9 @@ from pathlib import Path from django.conf import settings from django.core.files.storage import FileSystemStorage -from ..choices import Val, YouTube_SourceType media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') -_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) - - -def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): - element = nfo.makeelement(label, attrs) - element.text = text - element.tail = tail + (char * indent) - return element def get_media_file_path(instance, filename): diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py new file mode 100644 index 00000000..96539dbe --- /dev/null +++ b/tubesync/sync/models/_private.py @@ -0,0 +1,12 @@ +from ..choices import Val, YouTube_SourceType + + +_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) + + +def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2): + element = nfo.makeelement(label, attrs) + element.text = text + element.tail = tail + (char * indent) + return element + diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index eed6103a..92e2b23b 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -26,11 +26,11 @@ from ..matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from ..choices import ( Val, Fallback, MediaState, SourceResolution, YouTube_AudioCodec, YouTube_VideoCodec) -from .source import Source -from .misc import ( - media_file_storage, _srctype_dict, _nfo_element, - get_media_thumb_path, get_media_file_path, +from ._migrations import ( + media_file_storage, get_media_thumb_path, get_media_file_path, ) +from ._private import _srctype_dict, _nfo_element +from .source import Source class Media(models.Model): diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 7ab9ea4b..74f75278 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -20,7 +20,8 @@ from ..youtube import ( get_media_info as get_youtube_media_info, get_channel_image_info as get_youtube_channel_image_info, ) -from .misc import _srctype_dict, media_file_storage +from ._migrations import media_file_storage +from ._private import _srctype_dict class Source(db.models.Model): From 5c483d810c1afd7a850ce9e54fcb1533693d9618 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 10 May 2025 04:31:14 -0400 Subject: [PATCH 384/454] Re-indent imports --- tubesync/sync/models/media.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 92e2b23b..daaf723d 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -15,17 +15,27 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException -from common.utils import ( clean_filename, clean_emoji, - django_queryset_generator as qs_gen, ) -from ..youtube import ( get_media_info as get_youtube_media_info, - download_media as download_youtube_media) -from ..utils import (seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote, - multi_key_sort) -from ..matching import (get_best_combined_format, get_best_audio_format, - get_best_video_format) -from ..choices import ( Val, Fallback, MediaState, SourceResolution, - YouTube_AudioCodec, YouTube_VideoCodec) +from common.utils import ( + clean_filename, clean_emoji, + django_queryset_generator as qs_gen, +) +from ..youtube import ( + get_media_info as get_youtube_media_info, + download_media as download_youtube_media, +) +from ..utils import ( + seconds_to_timestr, parse_media_format, filter_response, + write_text_file, mkdir_p, directory_and_stem, glob_quote, + multi_key_sort, +) +from ..matching import ( + get_best_combined_format, + get_best_audio_format, get_best_video_format, +) +from ..choices import ( + Val, Fallback, MediaState, SourceResolution, + YouTube_AudioCodec, YouTube_VideoCodec, +) from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) From 12cdcdec96b48284f1a957b6ff11ec98b6744b31 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 05:21:26 -0400 Subject: [PATCH 385/454] Use a widget to select `when` in the browser --- tubesync/sync/forms.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index c0bfd13f..6df609da 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -49,6 +49,13 @@ class ScheduleTaskForm(forms.Form): when = forms.DateTimeField( label=_('When the task should run'), required=True, + #widget=forms.SplitDateTimeWidget( + # date_attrs={'type': 'date'}, + # time_attrs={'type': 'time'}, + #), + widget=forms.DateTimeInput( + attrs={'type': 'datetime-local'}, + ), ) From 3f5bd0746eb8bcdf21a1202e1953a8e7d52cba70 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 05:57:01 -0400 Subject: [PATCH 386/454] Adjust the description and button --- tubesync/sync/templates/sync/task-schedule.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/templates/sync/task-schedule.html b/tubesync/sync/templates/sync/task-schedule.html index 8f387b4a..63af2eb3 100644 --- a/tubesync/sync/templates/sync/task-schedule.html +++ b/tubesync/sync/templates/sync/task-schedule.html @@ -15,8 +15,8 @@ other tasks are taking to complete the assigned work.

- This will change the time that the task is requesting to be run - to the current time, plus some number of seconds. + This will change the time that the task is requesting to be the + current time, or a chosen future time.

@@ -26,7 +26,7 @@ {% include 'simpleform.html' with form=form %}
- +
From 05eb5cdeb0677805a848ee064326d03e8f2c5a4c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 06:48:46 -0400 Subject: [PATCH 387/454] Handle the timestamp from the URL correctly --- tubesync/sync/views.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 692675a0..dcf8e3e7 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1026,18 +1026,16 @@ class TaskScheduleView(FormView, SingleObjectMixin): def dispatch(self, request, *args, **kwargs): self.object = self.get_object() - self.timestamp = kwargs.get('timestamp', '') + self.timestamp = kwargs.get('timestamp') try: - self.timestamp = int(self.timestamp, 10) - except (TypeError, ValueError): - self.timestamp = None - else: - try: - self.when = timestamp_to_datetime(self.timestamp) - except AssertionError: - self.when = None + self.when = timestamp_to_datetime(self.timestamp) + except AssertionError: + self.when = None if self.when is None: self.when = timezone.now() + # Use the next minute and zero seconds + # The web browser does not select seconds by default + self.when = self.when.replace(second=0) + timezone.timedelta(minutes=1) return super().dispatch(request, *args, **kwargs) def get_initial(self): From 5bf4eeac488cda6881b755780b83daf9b57f9241 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 06:50:58 -0400 Subject: [PATCH 388/454] The split entry doesn't work because you can't strip a list --- tubesync/sync/forms.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 6df609da..2d03e4ea 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -49,10 +49,6 @@ class ScheduleTaskForm(forms.Form): when = forms.DateTimeField( label=_('When the task should run'), required=True, - #widget=forms.SplitDateTimeWidget( - # date_attrs={'type': 'date'}, - # time_attrs={'type': 'time'}, - #), widget=forms.DateTimeInput( attrs={'type': 'datetime-local'}, ), From af14930a077decfa1f76fabd5629a54ba460a16d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 08:16:53 -0400 Subject: [PATCH 389/454] Clean up extra metadata rows --- tubesync/sync/signals.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 657622ac..62f104bb 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -2,6 +2,7 @@ from functools import partial from pathlib import Path from tempfile import TemporaryDirectory from django.conf import settings +from django.db import IntegrityError from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.transaction import on_commit from django.dispatch import receiver @@ -439,9 +440,20 @@ def media_post_delete(sender, instance, **kwargs): skipped_media.skip = True skipped_media.manual_skip = True skipped_media.save() - Metadata.objects.filter( + # Re-use the old metadata if it exists + instance_qs = Metadata.objects.filter( media__isnull=True, site=old_metadata.get(site_field) or 'Youtube', key=skipped_media.key, - ).update(media=skipped_media) + ) + try: + instance_qs.update(media=skipped_media) + except IntegrityError: + # Delete the new metadata + Metadata.objects.filter(media=skipped_media).delete() + try: + instance_qs.update(media=skipped_media) + except IntegrityError: + # Delete the old metadata if it still failed + instance_qs.delete() From eace2bf45b3f3c1317466f893840dc028d6526a7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 13:36:00 -0400 Subject: [PATCH 390/454] Do not allow `run_at` to be in the past --- tubesync/sync/views.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index dcf8e3e7..c89041a3 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1016,15 +1016,18 @@ class TaskScheduleView(FormView, SingleObjectMixin): model = Task errors = dict( invalid_when=_('The type ({}) was incorrect.'), + when_before_now=_('The date and time must be in the future.'), ) def __init__(self, *args, **kwargs): + self.now = timezone.now() self.object = None self.timestamp = None self.when = None super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): + self.now = timezone.now() self.object = self.get_object() self.timestamp = kwargs.get('timestamp') try: @@ -1032,7 +1035,7 @@ class TaskScheduleView(FormView, SingleObjectMixin): except AssertionError: self.when = None if self.when is None: - self.when = timezone.now() + self.when = self.now # Use the next minute and zero seconds # The web browser does not select seconds by default self.when = self.when.replace(second=0) + timezone.timedelta(minutes=1) @@ -1040,11 +1043,13 @@ class TaskScheduleView(FormView, SingleObjectMixin): def get_initial(self): initial = super().get_initial() + initial['now'] = self.now initial['when'] = self.when return initial def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) + data['now'] = self.now data['when'] = self.when return data @@ -1059,24 +1064,28 @@ class TaskScheduleView(FormView, SingleObjectMixin): def form_valid(self, form): max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) - now = timezone.now() when = form.cleaned_data.get('when') - if not isinstance(when, now.__class__): + if not isinstance(when, self.now.__class__): form.add_error( 'when', ValidationError( - errors['invalid_when'].format( + self.errors['invalid_when'].format( type(when), ), ), ) + if when < self.now: + form.add_error( + 'when', + ValidationError(self.errors['when_before_now']), + ) if form.errors: return super().form_invalid(form) self.object.attempts = max_attempts // 2 - self.object.run_at = when + self.object.run_at = max(self.now, when) self.object.save() return super().form_valid(form) From 8be28b25bd5de60a3aec11dbdfe2da0724b408df Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 11 May 2025 13:46:34 -0400 Subject: [PATCH 391/454] Add a `now` input --- tubesync/sync/forms.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 2d03e4ea..39b1b606 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -46,6 +46,17 @@ class ResetTasksForm(forms.Form): class ScheduleTaskForm(forms.Form): + now = forms.DateTimeField( + label=_('The current date and time'), + required=False, + widget=forms.DateTimeInput( + attrs={ + 'type': 'datetime-local', + 'readonly': 'true', + }, + ), + ) + when = forms.DateTimeField( label=_('When the task should run'), required=True, From 01a44f74cf2ba585066e521b6319e03f2c16884b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 13 May 2025 04:52:10 -0400 Subject: [PATCH 392/454] Unpin Django --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 2976db2e..bf53b4bf 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,7 @@ verify_ssl = true autopep8 = "*" [packages] -django = "<5.2" +django = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" From 25af3e340732dc234f2f97c319a2c947ce319342 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 13 May 2025 05:24:22 -0400 Subject: [PATCH 393/454] Bump version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 391a209d..68bca98b 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.1' +VERSION = '0.15.2' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From aa27e1c22b6db95ce85bc105b1aec353a3b9f3e5 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:09:09 -0400 Subject: [PATCH 394/454] Adjust Python versions --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9edc7225..e3fee696 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python-version }} @@ -102,7 +102,7 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* - name: Run Django tests - run: cd tubesync && python3 manage.py test --verbosity=2 + run: cd tubesync && python3 -W error manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From 6c10a1da5b6ac7809a897b633c4dd61b908e021c Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:38:16 -0400 Subject: [PATCH 395/454] Check the Django version before using a removed setting --- tubesync/tubesync/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 68bca98b..06853cbf 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,3 +1,4 @@ +from django import VERSION as DJANGO_VERSION from pathlib import Path from common.utils import getenv @@ -99,7 +100,9 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' TIME_ZONE = getenv('TZ', 'UTC') USE_I18N = True -USE_L10N = True +# Removed in Django 5.0 +if DJANGO_VERSION[0:3] < (5, 0, 0): + USE_L10N = True USE_TZ = True From 406ba8a8cd0728840860bb67e1a9e85ebb268ddd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:47:32 -0400 Subject: [PATCH 396/454] Set `USE_L10N` to `True` only before Django 4 --- tubesync/tubesync/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 06853cbf..7f5922ae 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -100,8 +100,9 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' TIME_ZONE = getenv('TZ', 'UTC') USE_I18N = True -# Removed in Django 5.0 -if DJANGO_VERSION[0:3] < (5, 0, 0): +# Removed in Django 5.0, set to True by default in Django 4.0 +# https://docs.djangoproject.com/en/4.1/releases/4.0/#localization +if DJANGO_VERSION[0:3] < (4, 0, 0): USE_L10N = True USE_TZ = True From 49340701830bd4fa7b5005f5167c5e4549ae38c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 10:58:42 -0400 Subject: [PATCH 397/454] Explicitly pass `assume_scheme` --- tubesync/sync/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 39b1b606..655b93be 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -12,7 +12,8 @@ class ValidateSourceForm(forms.Form): ) source_url = forms.URLField( label=_('Source URL'), - required=True + required=True, + assume_scheme='http', # Silence RemovedInDjango60Warning ) From b6466c9b0eb4bf818e04bafad5de92ad2a5c7dc9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:12:58 -0400 Subject: [PATCH 398/454] Always use the UTC time zone --- tubesync/common/timestamp.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/common/timestamp.py b/tubesync/common/timestamp.py index df7b2f13..d8b69178 100644 --- a/tubesync/common/timestamp.py +++ b/tubesync/common/timestamp.py @@ -1,8 +1,8 @@ import datetime -posix_epoch = datetime.datetime.utcfromtimestamp(0) utc_tz = datetime.timezone.utc +posix_epoch = datetime.datetime.fromtimestamp(0, utc_tz) def add_epoch(seconds): @@ -13,10 +13,9 @@ def add_epoch(seconds): def subtract_epoch(arg_dt, /): assert isinstance(arg_dt, datetime.datetime) - epoch = posix_epoch.astimezone(utc_tz) utc_dt = arg_dt.astimezone(utc_tz) - return utc_dt - epoch + return utc_dt - posix_epoch def datetime_to_timestamp(arg_dt, /, *, integer=True): timestamp = subtract_epoch(arg_dt).total_seconds() From 0cf25a1bc8491ad58e4a4e9204e2267f025b37fa Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:29:05 -0400 Subject: [PATCH 399/454] `assume_scheme` was added in Django 5 --- tubesync/sync/forms.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index 655b93be..e46b740f 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -1,8 +1,14 @@ -from django import forms +from django import forms, VERSION as DJANGO_VERSION from django.utils.translation import gettext_lazy as _ +if DJANGO_VERSION[0:3] < (5, 0, 0): + _assume_scheme = dict() +else: + # Silence RemovedInDjango60Warning + _assume_scheme = dict(assume_scheme='http') + class ValidateSourceForm(forms.Form): source_type = forms.CharField( @@ -13,7 +19,7 @@ class ValidateSourceForm(forms.Form): source_url = forms.URLField( label=_('Source URL'), required=True, - assume_scheme='http', # Silence RemovedInDjango60Warning + **_assume_scheme, ) From b901f6f08c925d1ebb5a09bd68721ce3ba0e8aba Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:39:45 -0400 Subject: [PATCH 400/454] Run `collectstatic` before `test` --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3fee696..27240f3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -101,8 +101,9 @@ jobs: cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* + cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Run Django tests - run: cd tubesync && python3 -W error manage.py test --verbosity=2 + run: cd tubesync && python3 -W default manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From b99cb61cdb47758e9fa9579214092f032f1f651d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 11:58:29 -0400 Subject: [PATCH 401/454] Don't write `.pyc` files --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 27240f3e..f2a341b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Run Django tests - run: cd tubesync && python3 -W default manage.py test --verbosity=2 + run: cd tubesync && python3 -B -W default manage.py test --verbosity=2 containerise: if: ${{ !cancelled() && 'success' == needs.info.result }} From 06591a002075dfebe4b8e8a188ca08880a37ba6e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 12:46:57 -0400 Subject: [PATCH 402/454] Remove `Path.is_relative_to` patch for Python `3.8` --- tubesync/sync/signals.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 62f104bb..790ce1c2 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -22,20 +22,6 @@ from .filtering import filter_media from .choices import Val, YouTube_SourceType -def is_relative_to(self, *other): - """Return True if the path is relative to another path or False. - """ - try: - self.relative_to(*other) - return True - except ValueError: - return False - -# patch Path for Python 3.8 -if not hasattr(Path, 'is_relative_to'): - Path.is_relative_to = is_relative_to - - @receiver(pre_save, sender=Source) def source_pre_save(sender, instance, **kwargs): # Triggered before a source is saved, if the schedule has been updated recreate From 1c078322ef44e97a61136e044291870cadad6881 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 12:49:48 -0400 Subject: [PATCH 403/454] Remove Python `3.9` as unsupported --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f2a341b8..b7de46be 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python-version }} From 1142fb6844b55ff84c397a044992564b6ecb42ee Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 14 May 2025 13:04:58 -0400 Subject: [PATCH 404/454] Update README.md --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 502abf3a..2ea83c54 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ services: ## Optional authentication -Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and +Available in `v1.0` (or `:latest`) and later. If you want to enable a basic username and password to be required to access the TubeSync dashboard you can set them with the following environment variables: @@ -188,6 +188,14 @@ $ docker pull ghcr.io/meeb/tubesync:v[number] Back-end updates such as database migrations should be automatic. +> [!IMPORTANT] +> `MariaDB` was not automatically upgraded for `UUID` column types. +> To see what changes are needed, you can run: +> ```bash +> docker exec -it tubesync python3 /app/manage.py fix-mariadb --dry-run --uuid-columns +> ``` +> Removing the `--dry-run` will attempt to execute those statements using the configured database connection. + # Moving, backing up, etc. @@ -349,7 +357,7 @@ and you can probably break things by playing in the admin. If you still want to it you can run: ```bash -$ docker exec -ti tubesync python3 /app/manage.py createsuperuser +$ docker exec -it tubesync python3 /app/manage.py createsuperuser ``` And follow the instructions to create an initial Django superuser, once created, you @@ -415,7 +423,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.10) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From be1617979f44d8703176a111dc4c22cec6543ae9 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 15 May 2025 04:09:14 -0400 Subject: [PATCH 405/454] Use `db.connection.ensure_connection()` --- tubesync/sync/management/commands/fix-mariadb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index c3f2287a..58ad2202 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -24,6 +24,7 @@ def SQLTable(arg_table): needle = arg_table if needle.startswith('new__'): needle = arg_table[len('new__'):] + db.connection.ensure_connection() valid_table_name = ( needle in new_tables and arg_table in db_tables(include_views=False) @@ -122,6 +123,7 @@ class Command(BaseCommand): + f': {db.connection.vendor}' ) + db.connection.ensure_connection() db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and From add42bd87c3f90d08750ad3d02ad12e0d261ecce Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 15 May 2025 04:43:59 -0400 Subject: [PATCH 406/454] Use a migration that gives the output we search for --- tubesync/sync/management/commands/fix-mariadb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 58ad2202..9b21d2df 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -222,7 +222,7 @@ class Command(BaseCommand): at_31, err_31, out_31 = check_migration_status( '0031_metadata_metadataformat' ) at_31s, err_31s, out_31s = check_migration_status( '0031_squashed_metadata_metadataformat' ) after_31, err_31a, out_31a = check_migration_status( - '0030_alter_source_source_vcodec', + '0031_metadata_metadataformat', needle='Undo Rename table for metadata to sync_media_metadata', ) From 42456f6389a74e1c0b8df5f9929f766aa408664c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 04:21:33 -0400 Subject: [PATCH 407/454] Update release.yaml --- .github/workflows/release.yaml | 88 ++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9e2992d7..dea6c23f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,31 +8,89 @@ on: types: [published] jobs: - containerise: + info: runs-on: ubuntu-latest + outputs: + ffmpeg-date: ${{ steps.jq.outputs.FFMPEG_DATE }} + ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} + ffmpeg-version: ${{ steps.jq.outputs.FFMPEG_VERSION }} + lowercase-github-actor: ${{ steps.github-actor.outputs.lowercase }} + lowercase-github-repository_owner: ${{ steps.github-repository_owner.outputs.lowercase }} + tag: ${{ steps.tag.outputs.tag }} + ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} + ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }} steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - name: Get tag id: tag uses: dawidd6/action-get-tag@v1 - - uses: docker/build-push-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Log into GitHub Container Registry - run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin - - name: Lowercase github username for ghcr - id: string - uses: ASzc/change-string-case-action@v1 + - uses: actions/checkout@v4 + - name: Lowercase github username + id: github-actor + uses: ./.github/actions/string-case with: string: ${{ github.actor }} + - name: Lowercase github repository owner + id: github-repository_owner + uses: ./.github/actions/string-case + with: + string: ${{ github.repository_owner }} + - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI + id: ffmpeg + uses: ./.github/actions/FFmpeg + - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI + id: yt-dlp + uses: ./.github/actions/yt-dlp + - name: Set outputs with jq + id: jq + run: | + cat >| .ffmpeg.releases.json <<'EOF' + ${{ steps.ffmpeg.outputs.releases }} + EOF + mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; + { + var='FFMPEG_DATE' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ; + jq -r "${jq_arg}" -- .ffmpeg.releases.json ; + close_ml_var "${delim}" "${var}" ; + + ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )" + + var='FFMPEG_VERSION' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ; + jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ; + close_ml_var "${delim}" "${var}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" + cat -v "${GITHUB_OUTPUT}" + rm -v -f .ffmpeg.releases.json + + containerise: + needs: ['info'] + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log into GitHub Container Registry + run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: platforms: linux/amd64,linux/arm64 push: true - tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} - cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} - cache-to: type=inline + tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:${{ needs.info.outputs.tag }} + cache-from: | + type=registry,ref=ghcr.io/${{ needs.info.outputs.lowercase-github-repository_owner }}/${{ env.IMAGE_NAME }}:latest + type=gha build-args: | IMAGE_NAME=${{ env.IMAGE_NAME }} + FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }} + FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }} + YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} From e0e81c3a2c39742c6f8d88d92c43ff0ca36b53fa Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 04:59:55 -0400 Subject: [PATCH 408/454] Create get-tag/action.yml --- .github/actions/get-tag/action.yml | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/actions/get-tag/action.yml diff --git a/.github/actions/get-tag/action.yml b/.github/actions/get-tag/action.yml new file mode 100644 index 00000000..59ab43f3 --- /dev/null +++ b/.github/actions/get-tag/action.yml @@ -0,0 +1,41 @@ +name: Get tag +description: Get tag name from GITHUB_REF environment variable +inputs: + strip_v: + required: false + default: false + description: Whether to strip "v" from the tag or not +outputs: + tag: + value: ${{ steps.set.outputs.tag }} + description: Git tag name + +runs: + using: 'composite' + steps: + - name: Set outputs + id: 'set' + env: + INPUT_STRIP_V '${{ inputs.strip_v }}' + shell: 'bash' + run: | + tag="${GITHUB_REF}" + printf -- 'Manipulating string: %s\n' "${tag}" + test -n "${tag}" || exit 1 + + case "${tag}" in + (refs/tags/*) tag="${tag#refs/tags/}" ;; + (*) printf -- 'Not a tag ref\n' ; exit 2 ;; + esac + + if [ 'true' = "${INPUT_STRIP_V,,}" ] + then + tag="${tag#[Vv]}" + fi + + set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ; + + set_sl_var tag "${tag}" >> "${GITHUB_OUTPUT}" + + set_sl_var 'tag ' " ${tag}" + From 4d179fbd427c0a559bd7cd634f346f50f16560c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 05:03:24 -0400 Subject: [PATCH 409/454] Use the local `get-tag` action --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dea6c23f..621c31a5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,10 +20,10 @@ jobs: ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }} steps: + - uses: actions/checkout@v4 - name: Get tag id: tag - uses: dawidd6/action-get-tag@v1 - - uses: actions/checkout@v4 + uses: ./.github/actions/get-tag - name: Lowercase github username id: github-actor uses: ./.github/actions/string-case From 7923d03387aca14ef0c50c29e464b00aff726297 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 05:09:15 -0400 Subject: [PATCH 410/454] fixup: missing colon --- .github/actions/get-tag/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get-tag/action.yml b/.github/actions/get-tag/action.yml index 59ab43f3..7f554c70 100644 --- a/.github/actions/get-tag/action.yml +++ b/.github/actions/get-tag/action.yml @@ -16,7 +16,7 @@ runs: - name: Set outputs id: 'set' env: - INPUT_STRIP_V '${{ inputs.strip_v }}' + INPUT_STRIP_V: '${{ inputs.strip_v }}' shell: 'bash' run: | tag="${GITHUB_REF}" From 9de2728f35cc5dbc4a2a5e4e44d11194cd2bd04b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 06:50:03 -0400 Subject: [PATCH 411/454] Create media__tasks.py + `wait_for_premiere` --- tubesync/sync/models/media__tasks.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tubesync/sync/models/media__tasks.py diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py new file mode 100644 index 00000000..35f8595f --- /dev/null +++ b/tubesync/sync/models/media__tasks.py @@ -0,0 +1,22 @@ +from django.utils import timezone + + +def wait_for_premiere(self): + hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) + + in_hours = None + if self.has_metadata or not self.published: + return (False, in_hours,) + + now = timezone.now() + if self.published < now: + in_hours = 0 + self.manual_skip = False + self.skip = False + else: + in_hours = hours(self.published - now) + self.manual_skip = True + self.title = _(f'Premieres in {in_hours} hours') + + return (True, in_hours,) + From 9ee28befe0ffcfe0ad52cd04f0232d820fc2c7d1 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 06:54:42 -0400 Subject: [PATCH 412/454] Add `wait_for_premiere` function --- tubesync/sync/models/media.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index daaf723d..3529d889 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -40,6 +40,7 @@ from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) from ._private import _srctype_dict, _nfo_element +from .media__tasks import wait_for_premiere from .source import Source @@ -1219,3 +1220,7 @@ class Media(models.Model): except OSError as e: pass + +# add imported functions +Media.wait_for_premiere = wait_for_premiere + From 31fc6adc1190a98818ca880dbf58d0033b75c7fd Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 07:07:22 -0400 Subject: [PATCH 413/454] Use `Media.wait_for_premiere` function --- tubesync/sync/tasks.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index d29e8239..84eaad69 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -887,26 +887,19 @@ def rename_all_media_for_source(source_id): @background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.DB), remove_existing_tasks=True) def wait_for_media_premiere(media_id): - hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) - try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: raise InvalidTaskError(_('no such media')) from e else: - if media.has_metadata: + valid, hours = media.wait_for_premiere() + if not valid: return - now = timezone.now() - if media.published < now: - media.manual_skip = False - media.skip = False - # the download tasks start after the media is saved - else: - media.manual_skip = True - media.title = _(f'Premieres in {hours(media.published - now)} hours') + + if hours: task = get_media_premiere_task(media_id) if task: - update_task_status(task, f'available in {hours(media.published - now)} hours') + update_task_status(task, f'available in {hours} hours') save_model(media) From 7390a2b2b18c7d7ad23d9292d66145f21c7ca8a6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 07:41:37 -0400 Subject: [PATCH 414/454] Add `download_checklist` function --- tubesync/sync/models/media__tasks.py | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index 35f8595f..106a27c6 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -1,6 +1,52 @@ +from common.logger import log +from common.errors import ( + NoMetadataException, +) from django.utils import timezone +def download_checklist(self, skip_checks=False): + media = self + if skip_checks: + return True + + if not media.source.download_media: + log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' + f'the source {media.source} has since been marked to not download, ' + f'not downloading') + return False + if media.skip or media.manual_skip: + # Media was toggled to be skipped after the task was scheduled + log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' + f'it is now marked to be skipped, not downloading') + return False + # metadata is required to generate the proper filepath + if not media.has_metadata: + raise NoMetadataException('Metadata is not yet available.') + downloaded_file_exists = ( + media.downloaded and + media.has_metadata and + ( + media.media_file_exists or + media.filepath.exists() + ) + ) + if downloaded_file_exists: + # Media has been marked as downloaded before the download_media task was fired, + # skip it + log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' + f'it has already been marked as downloaded, not downloading again') + return False + max_cap_age = media.source.download_cap_date + published = media.published + if max_cap_age and published: + if published <= max_cap_age: + log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but ' + f'the source has a download cap and the media is now too old, ' + f'not downloading') + return False + + def wait_for_premiere(self): hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) From fb4185a9b14bbdf7946aaa9b10c3f98f77a8231e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 07:44:21 -0400 Subject: [PATCH 415/454] Add `Media.download_checklist` function --- tubesync/sync/models/media.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 3529d889..9b033338 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -40,7 +40,9 @@ from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) from ._private import _srctype_dict, _nfo_element -from .media__tasks import wait_for_premiere +from .media__tasks import ( + download_checklist, wait_for_premiere, +) from .source import Source @@ -1222,5 +1224,6 @@ class Media(models.Model): # add imported functions +Media.download_checklist = download_checklist Media.wait_for_premiere = wait_for_premiere From e507a63362ed72f775c6c88654b5ee20e15192ad Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 07:50:54 -0400 Subject: [PATCH 416/454] Use `Media.download_checklist` function --- tubesync/sync/tasks.py | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 84eaad69..ea0cb6a1 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -604,7 +604,7 @@ def download_media_thumbnail(media_id, url): @background(schedule=dict(priority=30, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def download_media(media_id): +def download_media(media_id, override=False): ''' Downloads the media to disk and attaches it to the Media instance. ''' @@ -613,41 +613,10 @@ def download_media(media_id): except Media.DoesNotExist as e: # Task triggered but the media no longer exists, do nothing raise InvalidTaskError(_('no such media')) from e - if not media.source.download_media: - log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' - f'the source {media.source} has since been marked to not download, ' - f'not downloading') - return - if media.skip or media.manual_skip: - # Media was toggled to be skipped after the task was scheduled - log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' - f'it is now marked to be skipped, not downloading') - return - # metadata is required to generate the proper filepath - if not media.has_metadata: - raise NoMetadataException('Metadata is not yet available.') - downloaded_file_exists = ( - media.downloaded and - media.has_metadata and - ( - media.media_file_exists or - media.filepath.exists() - ) - ) - if downloaded_file_exists: - # Media has been marked as downloaded before the download_media task was fired, - # skip it - log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' - f'it has already been marked as downloaded, not downloading again') - return - max_cap_age = media.source.download_cap_date - published = media.published - if max_cap_age and published: - if published <= max_cap_age: - log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but ' - f'the source has a download cap and the media is now too old, ' - f'not downloading') + else: + if not media.download_checklist(override): return + filepath = media.filepath container = format_str = None log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"') From 54b7e261f1f1f3a325015c1a8d72e2fe2e69cc69 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 08:16:18 -0400 Subject: [PATCH 417/454] Add `download_finished` function --- tubesync/sync/models/media__tasks.py | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index 106a27c6..fef043e3 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -1,3 +1,4 @@ +import os from common.logger import log from common.errors import ( NoMetadataException, @@ -47,6 +48,50 @@ def download_checklist(self, skip_checks=False): return False +def download_finished(self, format_str, container, downloaded_filepath=None): + media = self + if downloaded_filepath is None: + downloaded_filepath = self.filepath + filepath = Path(downloaded_filepath) + # Media has been downloaded successfully + log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' + f'"{filepath}"') + # Link the media file to the object and update info about the download + media.media_file.name = str(media.source.type_directory_path / media.filename) + media.downloaded = True + media.download_date = timezone.now() + media.downloaded_filesize = os.path.getsize(filepath) + media.downloaded_container = container + if '+' in format_str: + # Seperate audio and video streams + vformat_code, aformat_code = format_str.split('+') + aformat = media.get_format_by_code(aformat_code) + vformat = media.get_format_by_code(vformat_code) + media.downloaded_format = vformat['format'] + media.downloaded_height = vformat['height'] + media.downloaded_width = vformat['width'] + media.downloaded_audio_codec = aformat['acodec'] + media.downloaded_video_codec = vformat['vcodec'] + media.downloaded_container = container + media.downloaded_fps = vformat['fps'] + media.downloaded_hdr = vformat['is_hdr'] + else: + # Combined stream or audio-only stream + cformat_code = format_str + cformat = media.get_format_by_code(cformat_code) + media.downloaded_audio_codec = cformat['acodec'] + if cformat['vcodec']: + # Combined + media.downloaded_format = cformat['format'] + media.downloaded_height = cformat['height'] + media.downloaded_width = cformat['width'] + media.downloaded_video_codec = cformat['vcodec'] + media.downloaded_fps = cformat['fps'] + media.downloaded_hdr = cformat['is_hdr'] + else: + media.downloaded_format = 'audio' + + def wait_for_premiere(self): hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) From f77a6573229466430f6a02e47ec1fa7933159a0d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 08:18:21 -0400 Subject: [PATCH 418/454] Add `Media.download_finished` function --- tubesync/sync/models/media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 9b033338..6eb0ed76 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -41,7 +41,7 @@ from ._migrations import ( ) from ._private import _srctype_dict, _nfo_element from .media__tasks import ( - download_checklist, wait_for_premiere, + download_checklist, download_finished, wait_for_premiere, ) from .source import Source @@ -1225,5 +1225,6 @@ class Media(models.Model): # add imported functions Media.download_checklist = download_checklist +Media.download_finished = download_finished Media.wait_for_premiere = wait_for_premiere From 151640ae459c5c2d159e185a6e71298d39bd9ff9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 08:25:15 -0400 Subject: [PATCH 419/454] Use `Media.download_finished` function --- tubesync/sync/tasks.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ea0cb6a1..0467a4fd 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -649,42 +649,7 @@ def download_media(media_id, override=False): raise DownloadFailedException(err) # Media has been downloaded successfully - log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' - f'"{filepath}"') - # Link the media file to the object and update info about the download - media.media_file.name = str(media.source.type_directory_path / media.filename) - media.downloaded = True - media.download_date = timezone.now() - media.downloaded_filesize = os.path.getsize(filepath) - media.downloaded_container = container - if '+' in format_str: - # Seperate audio and video streams - vformat_code, aformat_code = format_str.split('+') - aformat = media.get_format_by_code(aformat_code) - vformat = media.get_format_by_code(vformat_code) - media.downloaded_format = vformat['format'] - media.downloaded_height = vformat['height'] - media.downloaded_width = vformat['width'] - media.downloaded_audio_codec = aformat['acodec'] - media.downloaded_video_codec = vformat['vcodec'] - media.downloaded_container = container - media.downloaded_fps = vformat['fps'] - media.downloaded_hdr = vformat['is_hdr'] - else: - # Combined stream or audio-only stream - cformat_code = format_str - cformat = media.get_format_by_code(cformat_code) - media.downloaded_audio_codec = cformat['acodec'] - if cformat['vcodec']: - # Combined - media.downloaded_format = cformat['format'] - media.downloaded_height = cformat['height'] - media.downloaded_width = cformat['width'] - media.downloaded_video_codec = cformat['vcodec'] - media.downloaded_fps = cformat['fps'] - media.downloaded_hdr = cformat['is_hdr'] - else: - media.downloaded_format = 'audio' + media.download_finished(format_str, container, filepath) save_model(media) # If selected, copy the thumbnail over as well if media.source.copy_thumbnails: From 346d66ff6921450a1b93735300e8da2db6ec869b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 08:27:02 -0400 Subject: [PATCH 420/454] Add a blank line --- tubesync/sync/models/media__tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index fef043e3..bf27813e 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -53,6 +53,7 @@ def download_finished(self, format_str, container, downloaded_filepath=None): if downloaded_filepath is None: downloaded_filepath = self.filepath filepath = Path(downloaded_filepath) + # Media has been downloaded successfully log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' f'"{filepath}"') From f0a4c9dea4db9bbe1ef458f0908d8e570e5ac8d5 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 08:49:04 -0400 Subject: [PATCH 421/454] Use storage location for `media_file` --- tubesync/sync/models/media__tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index bf27813e..d63f7b31 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -4,6 +4,7 @@ from common.errors import ( NoMetadataException, ) from django.utils import timezone +from ..choices import Val, SourceResolution def download_checklist(self, skip_checks=False): @@ -58,7 +59,7 @@ def download_finished(self, format_str, container, downloaded_filepath=None): log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' f'"{filepath}"') # Link the media file to the object and update info about the download - media.media_file.name = str(media.source.type_directory_path / media.filename) + self.media_file.name = str(filepath.relative_to(self.media_file.storage.location)) media.downloaded = True media.download_date = timezone.now() media.downloaded_filesize = os.path.getsize(filepath) @@ -90,7 +91,7 @@ def download_finished(self, format_str, container, downloaded_filepath=None): media.downloaded_fps = cformat['fps'] media.downloaded_hdr = cformat['is_hdr'] else: - media.downloaded_format = 'audio' + self.downloaded_format = Val(SourceResolution.AUDIO) def wait_for_premiere(self): From a68c82d975c083aee75e443fffefe805b3afabf7 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 18:37:11 -0400 Subject: [PATCH 422/454] Add `bgutil-ytdlp-pot-provider` plugin --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index aed25ce3..5d76b308 100644 --- a/Pipfile +++ b/Pipfile @@ -25,3 +25,4 @@ emoji = "*" brotli = "*" html5lib = "*" yt-dlp-get-pot = "*" +bgutil-ytdlp-pot-provider = "*" From 12aa6b413470a71bbf4c637ba6d1f581783efeaf Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 19:55:56 -0400 Subject: [PATCH 423/454] Use token_server.conf --- config/root/etc/nginx/nginx.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf index e6b99b68..e4e9140e 100644 --- a/config/root/etc/nginx/nginx.conf +++ b/config/root/etc/nginx/nginx.conf @@ -135,4 +135,7 @@ http { } } + # Proof-of-Origin Token Server + include /etc/nginx/token_server.conf; + } From 4abd768ba8c56e93ea3c15aa9c898b1a42f047fb Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 19:59:09 -0400 Subject: [PATCH 424/454] Create token_server.conf --- config/root/etc/nginx/token_server.conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 config/root/etc/nginx/token_server.conf diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf new file mode 100644 index 00000000..fbfd5979 --- /dev/null +++ b/config/root/etc/nginx/token_server.conf @@ -0,0 +1,15 @@ +upstream token_server { + server 127.0.0.2:4416 down; +} + +server { + + # Ports + listen 4416; + listen [::]:4416; + + location / { + proxy_pass http://token_server; + proxy_redirect default; + } +} From 8819a945966acedb6b510f4db0bd41e859dad5d9 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 21:32:54 -0400 Subject: [PATCH 425/454] Add `server_name` --- config/root/etc/nginx/token_server.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index fbfd5979..a49f3a3e 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -8,6 +8,9 @@ server { listen 4416; listen [::]:4416; + # Server domain name + server_name _; + location / { proxy_pass http://token_server; proxy_redirect default; From 5f797d091541f5251d1e374b474f76f707a98759 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 21:57:37 -0400 Subject: [PATCH 426/454] Add `YT_POT_BGUTIL_BASE_URL` environment variable --- config/root/etc/nginx/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf index e4e9140e..1b544f44 100644 --- a/config/root/etc/nginx/nginx.conf +++ b/config/root/etc/nginx/nginx.conf @@ -5,6 +5,8 @@ worker_processes auto; worker_cpu_affinity auto; pid /run/nginx.pid; +env YT_POT_BGUTIL_BASE_URL; + events { worker_connections 1024; } From 0919b03143ea379473c35f47045c1b6db5a05919 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 22:11:42 -0400 Subject: [PATCH 427/454] Set the scheme from `YT_POT_BGUTIL_BASE_URL` --- config/root/etc/nginx/token_server.conf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index a49f3a3e..057cade5 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -11,8 +11,18 @@ server { # Server domain name server_name _; + set_by_lua_block $pot_url_scheme { + local haystack = os.getenv('YT_POT_BGUTIL_BASE_URL') or '' + local needle = 'https://' + local scheme = 'http' + if haystack:sub(1, #needle) == needle then + scheme = 'https' + end + return scheme + } + location / { - proxy_pass http://token_server; + proxy_pass $pot_url_scheme://token_server; proxy_redirect default; } } From f7d3367a9ff284288a038228794706de3a657bc6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 23:37:23 -0400 Subject: [PATCH 428/454] Use the environment variable directly --- config/root/etc/nginx/token_server.conf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index 057cade5..6c0c4e3a 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -11,18 +11,19 @@ server { # Server domain name server_name _; - set_by_lua_block $pot_url_scheme { - local haystack = os.getenv('YT_POT_BGUTIL_BASE_URL') or '' - local needle = 'https://' - local scheme = 'http' - if haystack:sub(1, #needle) == needle then - scheme = 'https' + set_by_lua_block $pot_url { + local default = 'http://token_server' + local url = os.getenv('YT_POT_BGUTIL_BASE_URL') + if not url then + return default end - return scheme + if #url and url:find('://') then + return url + end + return default } location / { - proxy_pass $pot_url_scheme://token_server; - proxy_redirect default; + proxy_pass $pot_url; } } From 14877a8c7d1c2b9a7113b1af4717346010e62e51 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 16 May 2025 23:59:13 -0400 Subject: [PATCH 429/454] Set `YT_POT_BGUTIL_BASE_URL` for the user --- config/root/etc/s6-overlay/s6-rc.d/nginx/run | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index 63653343..de89b819 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -2,4 +2,14 @@ cd / +https="${TUBESYNC_POT_HTTPS:+https}" +ip_address="${TUBESYNC_POT_IPADDR}" +port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}" + +if [ -n "${ip_address}" ] +then + YT_POT_BGUTIL_BASE_URL="${https:-http}://${ip_address}${port}" + export YT_POT_BGUTIL_BASE_URL +fi + exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr From 98f3bba9e8306631d18280172a4d8261b8491219 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 00:31:55 -0400 Subject: [PATCH 430/454] Support `--link` variables --- config/root/etc/s6-overlay/s6-rc.d/nginx/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index de89b819..fff967d1 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -3,7 +3,8 @@ cd / https="${TUBESYNC_POT_HTTPS:+https}" -ip_address="${TUBESYNC_POT_IPADDR}" +ip_address="${TUBESYNC_POT_IPADDR:-${POTSERVER_PORT_4416_TCP_ADDR}}" +: "${TUBESYNC_POT_PORT:=${POTSERVER_PORT_4416_TCP_PORT}}" port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}" if [ -n "${ip_address}" ] From ea911e4bf91645e987f501c6f9d7def3930fe2a0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 01:49:32 -0400 Subject: [PATCH 431/454] Bump version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 7f5922ae..798bd252 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -8,7 +8,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.2' +VERSION = '0.15.3' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From b93b4e1d47a49d94f2d71a5eb8402e64a9d7461b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 08:22:56 -0400 Subject: [PATCH 432/454] Add a step for `ruff check` --- .github/workflows/ci.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7de46be..332801e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,7 +94,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pipenv + pip install pipenv uv PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - name: Set up Django environment run: | @@ -102,6 +102,12 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* cd tubesync && python3 -B manage.py collectstatic --no-input --link + - name: Check with ruff + continue-on-error: true + run: | + cd tubesync && uvx ruff check --output-format github \ + --target-version py310 --isolated \ + --ignore E701,E722,E731 - name: Run Django tests run: cd tubesync && python3 -B -W default manage.py test --verbosity=2 From f753d25218c7acbc062f4f8ae896b7f37098f926 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 23:00:38 -0400 Subject: [PATCH 433/454] Use `uv` to install `pipenv` --- .github/workflows/ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 332801e9..54a0d553 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,9 +93,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pipenv uv - PIPENV_VERBOSITY=64 pipenv install --system --skip-lock + python -m pip install uv + PIPENV_VERBOSITY=64 uvx --no-config --no-managed-python -v --from 'pipenv==2022.12.19' pipenv install --system --skip-lock - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From 9953092ee33a4eb41350ae97d1694eead7f28d51 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 23:08:15 -0400 Subject: [PATCH 434/454] Also upgrade `pip` --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 54a0d553..4a295bf3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,7 +93,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install uv + python -m pip install --upgrade pip uv PIPENV_VERBOSITY=64 uvx --no-config --no-managed-python -v --from 'pipenv==2022.12.19' pipenv install --system --skip-lock - name: Set up Django environment run: | From 31acdc8a73fa2274c959ecc7895f3f3b0f5c9af4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 23:15:56 -0400 Subject: [PATCH 435/454] Use `uv pip` --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a295bf3..a3939e11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,8 +93,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip uv - PIPENV_VERBOSITY=64 uvx --no-config --no-managed-python -v --from 'pipenv==2022.12.19' pipenv install --system --skip-lock + python -m pip install uv + uv -v pip install 'pipenv==2022.12.19' + PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From b05b594a3d10bfa395c0d8256f3d457a45959723 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 23:26:57 -0400 Subject: [PATCH 436/454] Use `--system` flag --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a3939e11..8de863b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,7 +94,8 @@ jobs: - name: Install dependencies run: | python -m pip install uv - uv -v pip install 'pipenv==2022.12.19' + uv --no-config --no-managed-python --no-progress -v \ + pip install --system --strict 'pipenv==2022.12.19' PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - name: Set up Django environment run: | From eb26fbca8925daaef5e4f54025dd0a8b2a806563 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 23:31:11 -0400 Subject: [PATCH 437/454] Unpin `pipenv` --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8de863b2..34b73468 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,7 +95,7 @@ jobs: run: | python -m pip install uv uv --no-config --no-managed-python --no-progress -v \ - pip install --system --strict 'pipenv==2022.12.19' + pip install --system --strict pipenv PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - name: Set up Django environment run: | From f6b56c1c4e606c818b189cba8697593d7adfa21c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 00:19:29 -0400 Subject: [PATCH 438/454] Install with `uv` from requirements.txt --- .github/workflows/ci.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 34b73468..d61d9ece 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,7 +96,12 @@ jobs: python -m pip install uv uv --no-config --no-managed-python --no-progress -v \ pip install --system --strict pipenv - PIPENV_VERBOSITY=64 pipenv install --system --skip-lock + PIPENV_VERBOSITY=64 pipenv lock + PIPENV_VERBOSITY=64 pipenv requirements --hash > requirements.txt + cat -v -n requirements.txt + uv --no-config --no-managed-python --no-progress -v \ + pip install --system --strict --requirements requirements.txt + #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From 3a326a526b3783935e210c31a3af3a850f6e9e94 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 00:29:50 -0400 Subject: [PATCH 439/454] Do not output hashes --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d61d9ece..690354b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,7 +97,7 @@ jobs: uv --no-config --no-managed-python --no-progress -v \ pip install --system --strict pipenv PIPENV_VERBOSITY=64 pipenv lock - PIPENV_VERBOSITY=64 pipenv requirements --hash > requirements.txt + PIPENV_VERBOSITY=64 pipenv requirements > requirements.txt cat -v -n requirements.txt uv --no-config --no-managed-python --no-progress -v \ pip install --system --strict --requirements requirements.txt From 9641cc579b5aa03f98dc2f4668c12fe49eac77a5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 00:57:12 -0400 Subject: [PATCH 440/454] Use `pipenv` for install --- .github/workflows/ci.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 690354b2..d967c585 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,12 +96,13 @@ jobs: python -m pip install uv uv --no-config --no-managed-python --no-progress -v \ pip install --system --strict pipenv - PIPENV_VERBOSITY=64 pipenv lock - PIPENV_VERBOSITY=64 pipenv requirements > requirements.txt - cat -v -n requirements.txt - uv --no-config --no-managed-python --no-progress -v \ - pip install --system --strict --requirements requirements.txt - #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock + #PIPENV_VERBOSITY=64 pipenv lock + #PIPENV_VERBOSITY=64 pipenv requirements > requirements.txt + #cat -v -n requirements.txt + #uv --no-config --no-managed-python --no-progress -v \ + # pip install --system --strict --requirements requirements.txt + PIPENV_VERBOSITY=64 pipenv install --system + pipenv requirements - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From e737a5443fa2b6c6f6e6e88d87cbc4c901e4b652 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:02:52 -0400 Subject: [PATCH 441/454] Use lock separately --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d967c585..a1f8d5ee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,10 +99,11 @@ jobs: #PIPENV_VERBOSITY=64 pipenv lock #PIPENV_VERBOSITY=64 pipenv requirements > requirements.txt #cat -v -n requirements.txt + PIPENV_VERBOSITY=64 pipenv install --system --skip-lock + pipenv lock + pipenv requirements #uv --no-config --no-managed-python --no-progress -v \ # pip install --system --strict --requirements requirements.txt - PIPENV_VERBOSITY=64 pipenv install --system - pipenv requirements - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From a76919aef411dcd64d7ab783aea271566e8ab063 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:09:57 -0400 Subject: [PATCH 442/454] Install with `uv` is faster --- .github/workflows/ci.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1f8d5ee..0f02ffd1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,14 +96,11 @@ jobs: python -m pip install uv uv --no-config --no-managed-python --no-progress -v \ pip install --system --strict pipenv - #PIPENV_VERBOSITY=64 pipenv lock - #PIPENV_VERBOSITY=64 pipenv requirements > requirements.txt - #cat -v -n requirements.txt - PIPENV_VERBOSITY=64 pipenv install --system --skip-lock pipenv lock - pipenv requirements - #uv --no-config --no-managed-python --no-progress -v \ - # pip install --system --strict --requirements requirements.txt + PIPENV_VERBOSITY=64 pipenv requirements | tee requirements.txt + #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock + uv --no-config --no-managed-python --no-progress -v \ + pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py From 71cf95cdefe046dfc6c596d5c42c3776f6548f1f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:14:11 -0400 Subject: [PATCH 443/454] Disable verbose output --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f02ffd1..3aa4d49b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,12 +94,12 @@ jobs: - name: Install dependencies run: | python -m pip install uv - uv --no-config --no-managed-python --no-progress -v \ + uv --no-config --no-managed-python --no-progress \ pip install --system --strict pipenv pipenv lock - PIPENV_VERBOSITY=64 pipenv requirements | tee requirements.txt + pipenv requirements | tee requirements.txt #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock - uv --no-config --no-managed-python --no-progress -v \ + uv --no-config --no-managed-python --no-progress \ pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | From 3fe1b54b33bec204b808ecbbd99d21490929209c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:18:49 -0400 Subject: [PATCH 444/454] Try without explicit `pipenv lock` --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3aa4d49b..65c3f87c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,7 +96,7 @@ jobs: python -m pip install uv uv --no-config --no-managed-python --no-progress \ pip install --system --strict pipenv - pipenv lock + #pipenv lock pipenv requirements | tee requirements.txt #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock uv --no-config --no-managed-python --no-progress \ From 294f07d4c467c0ba44bdc42092dd6c37962a2914 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:23:33 -0400 Subject: [PATCH 445/454] The explicit `pipenv lock` is needed --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 65c3f87c..3aa4d49b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -96,7 +96,7 @@ jobs: python -m pip install uv uv --no-config --no-managed-python --no-progress \ pip install --system --strict pipenv - #pipenv lock + pipenv lock pipenv requirements | tee requirements.txt #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock uv --no-config --no-managed-python --no-progress \ From b9efeeebf280799674f4086293c596868c3736fc Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:39:32 -0400 Subject: [PATCH 446/454] Do not upgrade to Django 5.3 automatically --- Pipfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index bf53b4bf..e4d6d053 100644 --- a/Pipfile +++ b/Pipfile @@ -7,13 +7,13 @@ verify_ssl = true autopep8 = "*" [packages] -django = "*" +django = "~= 5.2.1" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" gunicorn = "*" httptools = "*" -django-background-tasks = ">=1.2.8" +django-background-tasks = ">= 1.2.8" django-basicauth = "*" psycopg = {extras = ["binary", "pool"], version = "*"} mysqlclient = "*" From 7165491b25e30ae44f7656b9b4c255ed6b2f49a8 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 01:42:39 -0400 Subject: [PATCH 447/454] Remove the spaces for better log output --- Pipfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index e4d6d053..b466d84e 100644 --- a/Pipfile +++ b/Pipfile @@ -7,13 +7,13 @@ verify_ssl = true autopep8 = "*" [packages] -django = "~= 5.2.1" +django = "~=5.2.1" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" gunicorn = "*" httptools = "*" -django-background-tasks = ">= 1.2.8" +django-background-tasks = ">=1.2.8" django-basicauth = "*" psycopg = {extras = ["binary", "pool"], version = "*"} mysqlclient = "*" From 5dfe52452e2fd45af236e79a8a51461db88eef5d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 04:30:31 -0400 Subject: [PATCH 448/454] Use the step summary markdown file --- .github/workflows/ci.yaml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3aa4d49b..a1754c10 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -108,11 +108,30 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Check with ruff - continue-on-error: true + continue-on-error: false run: | - cd tubesync && uvx ruff check --output-format github \ - --target-version py310 --isolated \ - --ignore E701,E722,E731 + cd tubesync + # output formats: + # "full" | "concise" | "grouped" | + # "json" | "junit" | "github" | "gitlab" | + # "pylint" | "azure" + { + echo '# Output from `ruff check` for `tubesync`' + echo '' + echo '## pylint format' + echo '' + uvx --no-config --no-managed-python --no-progress --isolated \ + ruff check --exit-zero \ + --target-version py310 \ + --output-format pylint \ + --extend-select RUF100 \ + --ignore E701,E722,E731 + } >> "${GITHUB_STEP_SUMMARY}" + uvx --no-config --no-managed-python --no-progress --isolated \ + ruff check --exit-zero \ + --target-version py310 \ + --output-format github \ + --ignore E701,E722,E731 - name: Run Django tests run: cd tubesync && python3 -B -W default manage.py test --verbosity=2 From f182a41a219cbb80dd3b6be15f33d8c8f1b03698 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 04:46:14 -0400 Subject: [PATCH 449/454] Loop over output formats --- .github/workflows/ci.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1754c10..05debc82 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,16 +116,26 @@ jobs: # "json" | "junit" | "github" | "gitlab" | # "pylint" | "azure" { - echo '# Output from `ruff check` for `tubesync`' + echo '## Output from `ruff check` for `tubesync`' echo '' - echo '## pylint format' + echo '### Formats' + for fmt in full concise grouped pylint + do + echo '
' + echo ''"${fmt}"'' + echo '#### '"${fmt}"' output format' echo '' + echo '```' uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ --target-version py310 \ - --output-format pylint \ + --output-format "${fmt}" \ --extend-select RUF100 \ --ignore E701,E722,E731 + echo '```' + echo '
' + echo '' + done } >> "${GITHUB_STEP_SUMMARY}" uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ From 9c342c9ccc34c452d6b8406790652a160ec63874 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 04:50:41 -0400 Subject: [PATCH 450/454] Tweak the markdown --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05debc82..0fcf8054 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,11 +119,12 @@ jobs: echo '## Output from `ruff check` for `tubesync`' echo '' echo '### Formats' + echo '' for fmt in full concise grouped pylint do echo '
' echo ''"${fmt}"'' - echo '#### '"${fmt}"' output format' + echo '### '"${fmt}"' output format' echo '' echo '```' uvx --no-config --no-managed-python --no-progress --isolated \ From 4d036356e2a2aab2c4f6eee4590fe06c5dcf2452 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 04:58:12 -0400 Subject: [PATCH 451/454] Add a blank line after the summary --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0fcf8054..d7ce5080 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -124,7 +124,8 @@ jobs: do echo '
' echo ''"${fmt}"'' - echo '### '"${fmt}"' output format' + echo '' + echo '#### '"${fmt}"' output format' echo '' echo '```' uvx --no-config --no-managed-python --no-progress --isolated \ From 8e888b4182a1f454fe2a8c76d74623f612298529 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 05:06:32 -0400 Subject: [PATCH 452/454] Only quote `full` output --- .github/workflows/ci.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d7ce5080..884ea952 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,14 +127,16 @@ jobs: echo '' echo '#### '"${fmt}"' output format' echo '' - echo '```' + case "${fmt}" in (full) echo '```' ;; esac uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ --target-version py310 \ --output-format "${fmt}" \ --extend-select RUF100 \ --ignore E701,E722,E731 - echo '```' + echo '' + case "${fmt}" in (full) echo '```' ;; esac + echo '' echo '
' echo '' done From d502dcd01a58adefd077b7b7d07ea0d3ad5463ed Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 05:10:25 -0400 Subject: [PATCH 453/454] The output was better quoted --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 884ea952..7ae9e61d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,7 +127,7 @@ jobs: echo '' echo '#### '"${fmt}"' output format' echo '' - case "${fmt}" in (full) echo '```' ;; esac + echo '```' uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ --target-version py310 \ @@ -135,7 +135,7 @@ jobs: --extend-select RUF100 \ --ignore E701,E722,E731 echo '' - case "${fmt}" in (full) echo '```' ;; esac + echo '```' echo '' echo '
' echo '' From 46f1d7fc89068b9c11c221a4496cac459a934dc6 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 05:18:23 -0400 Subject: [PATCH 454/454] DRY the `ruff check` options --- .github/workflows/ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7ae9e61d..22bd31b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,6 +110,8 @@ jobs: - name: Check with ruff continue-on-error: false run: | + target_version='py310' + ignore_csv_list='E701,E722,E731' cd tubesync # output formats: # "full" | "concise" | "grouped" | @@ -130,10 +132,10 @@ jobs: echo '```' uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ - --target-version py310 \ + --target-version "${target_version}" \ --output-format "${fmt}" \ --extend-select RUF100 \ - --ignore E701,E722,E731 + --ignore "${ignore_csv_list}" echo '' echo '```' echo '' @@ -143,9 +145,9 @@ jobs: } >> "${GITHUB_STEP_SUMMARY}" uvx --no-config --no-managed-python --no-progress --isolated \ ruff check --exit-zero \ - --target-version py310 \ + --target-version "${target_version}" \ --output-format github \ - --ignore E701,E722,E731 + --ignore "${ignore_csv_list}" - name: Run Django tests run: cd tubesync && python3 -B -W default manage.py test --verbosity=2