From fa5bc3ecab913d7aeafc8a0c5a426d5873700f16 Mon Sep 17 00:00:00 2001 From: George Antoniadis Date: Thu, 8 Dec 2022 01:12:32 +0000 Subject: [PATCH 01/13] skip metadata and thumbnails once they are past the cutoff date --- tubesync/sync/signals.py | 4 ++-- tubesync/sync/tasks.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 9e417ffb..71a3476c 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -147,7 +147,7 @@ def media_post_save(sender, instance, created, **kwargs): instance.save() post_save.connect(media_post_save, sender=Media) # If the media is missing metadata schedule it to be downloaded - if not instance.metadata: + if not instance.metadata and not instance.skip: log.info(f'Scheduling task to download metadata for: {instance.url}') verbose_name = _('Downloading metadata for "{}"') download_media_metadata( @@ -159,7 +159,7 @@ def media_post_save(sender, instance, created, **kwargs): # If the media is missing a thumbnail schedule it to be downloaded if not instance.thumb_file_exists: instance.thumb = None - if not instance.thumb: + if not instance.thumb and not instance.skip: thumbnail_url = instance.thumbnail if thumbnail_url: log.info(f'Scheduling task to download thumbnail for: {instance.name} ' diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 34f13816..c290925e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -277,6 +277,11 @@ def download_media_thumbnail(media_id, url): except Media.DoesNotExist: # Task triggered but the media no longer exists, do nothing return + if media.skip: + # Media was toggled to be skipped after the task was scheduled + log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' + f'it is now marked to be skipped, not downloading thumbnail') + return width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) i = get_remote_image(url) From 43cf5329039d435780f502dc9226a2b9c00f3335 Mon Sep 17 00:00:00 2001 From: sparklesmcfadden Date: Sat, 21 Oct 2023 20:54:21 -0500 Subject: [PATCH 02/13] Adds option to delete files on disk when removing a media item from TubeSync --- .../migrations/0020_add_delete_files_on_disk.py | 17 +++++++++++++++++ tubesync/sync/models.py | 5 +++++ tubesync/sync/signals.py | 14 ++++++++++++++ tubesync/sync/templates/sync/source.html | 4 ++++ tubesync/sync/views.py | 6 +++--- 5 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tubesync/sync/migrations/0020_add_delete_files_on_disk.py diff --git a/tubesync/sync/migrations/0020_add_delete_files_on_disk.py b/tubesync/sync/migrations/0020_add_delete_files_on_disk.py new file mode 100644 index 00000000..4d331488 --- /dev/null +++ b/tubesync/sync/migrations/0020_add_delete_files_on_disk.py @@ -0,0 +1,17 @@ + Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0019_add_delete_removed_media'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index bb8c7233..05ca6fc9 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -292,6 +292,11 @@ class Source(models.Model): default=False, help_text=_('Delete media that is no longer on this playlist') ) + delete_files_on_disk = models.BooleanField( + _('delete files on disk'), + default=False, + help_text=_('Delete files on disk when they are removed from TubeSync') + ) source_resolution = models.CharField( _('source resolution'), max_length=8, diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index f27b452d..0c7a7a36 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -203,6 +203,20 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) + if instance.source.delete_files_on_disk: + if instance.thumb: + log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}') + delete_file(instance.thumb.path) + # Delete the media file if it exists + if instance.media_file: + filepath = instance.media_file.path + log.info(f'Deleting media for: {instance} path: {filepath}') + delete_file(filepath) + # Delete thumbnail copy if it exists + barefilepath, fileext = os.path.splitext(filepath) + thumbpath = f'{barefilepath}.jpg' + log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}') + delete_file(thumbpath) @receiver(post_delete, sender=Media) diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 22122e2c..c2a2cd49 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -118,6 +118,10 @@ Delete removed media Delete removed media
{% if source.delete_removed_media %}{% else %}{% endif %} + + + Delete files on disk + Delete files on disk
{% if source.delete_files_on_disk %}{% else %}{% endif %} {% if source.delete_old_media and source.days_to_keep > 0 %} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index e187cd85..f747a372 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -296,9 +296,9 @@ class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', - 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', - 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', + 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', + 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', + 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') errors = { From 46a43b968acb74964209a5417854e9fc11711556 Mon Sep 17 00:00:00 2001 From: administrator <7dn1yh5j@debauchez.fr> Date: Mon, 11 Dec 2023 02:29:57 +0100 Subject: [PATCH 03/13] Rework delete method to delete all files matching filename Remove Source folder if checkbox 'remove media' is checked --- ...sk.py => 0021_add_delete_files_on_disk.py} | 4 +-- tubesync/sync/signals.py | 26 ++++++++-------- .../sync/templates/sync/source-delete.html | 4 +-- tubesync/sync/views.py | 30 ++++++++++--------- 4 files changed, 32 insertions(+), 32 deletions(-) rename tubesync/sync/migrations/{0020_add_delete_files_on_disk.py => 0021_add_delete_files_on_disk.py} (85%) diff --git a/tubesync/sync/migrations/0020_add_delete_files_on_disk.py b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py similarity index 85% rename from tubesync/sync/migrations/0020_add_delete_files_on_disk.py rename to tubesync/sync/migrations/0021_add_delete_files_on_disk.py index 4d331488..5745d478 100644 --- a/tubesync/sync/migrations/0020_add_delete_files_on_disk.py +++ b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py @@ -1,11 +1,11 @@ - Generated by pac +# Generated by pac from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('sync', '0019_add_delete_removed_media'), + ('sync', '0020_auto_20231024_1825'), ] operations = [ diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 3d60424b..b9f88357 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -1,4 +1,5 @@ import os +import glob from django.conf import settings from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.dispatch import receiver @@ -74,6 +75,7 @@ def source_pre_delete(sender, instance, **kwargs): media.delete() + @receiver(post_delete, sender=Source) def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted @@ -222,20 +224,16 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) - if instance.source.delete_files_on_disk: - if instance.thumb: - log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}') - delete_file(instance.thumb.path) - # Delete the media file if it exists - if instance.media_file: - filepath = instance.media_file.path - log.info(f'Deleting media for: {instance} path: {filepath}') - delete_file(filepath) - # Delete thumbnail copy if it exists - barefilepath, fileext = os.path.splitext(filepath) - thumbpath = f'{barefilepath}.jpg' - log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}') - delete_file(thumbpath) + if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb): + # Delete all media files if it contains filename + filepath = instance.media_file.path if instance.media_file else instance.thumb.path + barefilepath, fileext = os.path.splitext(filepath) + # Get all files that start with the bare file path + all_related_files = glob.glob(f'{barefilepath}.*') + for file in all_related_files: + log.info(f'Deleting file for: {instance} path: {file}') + delete_file(file) + @receiver(post_delete, sender=Media) diff --git a/tubesync/sync/templates/sync/source-delete.html b/tubesync/sync/templates/sync/source-delete.html index bdc95208..ff4ef3b1 100644 --- a/tubesync/sync/templates/sync/source-delete.html +++ b/tubesync/sync/templates/sync/source-delete.html @@ -9,8 +9,8 @@

Are you sure you want to delete this source? Deleting a source is permanent. By default, deleting a source does not delete any saved media files. You can - tick the "also delete downloaded media" checkbox to also remove save - media when you delete the source. Deleting a source cannot be undone. + tick the "also delete downloaded media" checkbox to also remove directory {{ source.directory_path }} + when you delete the source. Deleting a source cannot be undone.

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 1eb5423f..ac65d5cc 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1,7 +1,9 @@ +import glob import os import json from base64 import b64decode import pathlib +import shutil import sys from django.conf import settings from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect @@ -435,14 +437,13 @@ class DeleteSourceView(DeleteView, FormMixin): source = self.get_object() for media in Media.objects.filter(source=source): if media.media_file: - # Delete the media file - delete_file(media.media_file.path) - # Delete thumbnail copy if it exists - delete_file(media.thumbpath) - # Delete NFO file if it exists - delete_file(media.nfopath) - # Delete JSON file if it exists - delete_file(media.jsonpath) + file_path = media.media_file.path + matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*') + for file in matching_files: + delete_file(file) + directory_path = source.directory_path + if os.path.exists(directory_path): + shutil.rmtree(directory_path, True) return super().post(request, *args, **kwargs) def get_success_url(self): @@ -653,12 +654,13 @@ class MediaSkipView(FormView, SingleObjectMixin): delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) # If the media file exists on disk, delete it if self.object.media_file_exists: - delete_file(self.object.media_file.path) - self.object.media_file = None - # If the media has an associated thumbnail copied, also delete it - delete_file(self.object.thumbpath) - # If the media has an associated NFO file with it, also delete it - delete_file(self.object.nfopath) + # Delete all files which contains filename + filepath = self.object.media_file.path + barefilepath, fileext = os.path.splitext(filepath) + # Get all files that start with the bare file path + all_related_files = glob.glob(f'{barefilepath}.*') + for file in all_related_files: + delete_file(file) # Reset all download data self.object.metadata = None self.object.downloaded = False From 46ba2593a2d7d756676d6e57e0924326ee1bb74e Mon Sep 17 00:00:00 2001 From: administrator <7dn1yh5j@debauchez.fr> Date: Mon, 11 Dec 2023 03:19:43 +0100 Subject: [PATCH 04/13] Removed non-pertinent source option --- .../migrations/0021_add_delete_files_on_disk.py | 17 ----------------- tubesync/sync/models.py | 5 ----- tubesync/sync/signals.py | 2 +- tubesync/sync/views.py | 2 +- 4 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 tubesync/sync/migrations/0021_add_delete_files_on_disk.py diff --git a/tubesync/sync/migrations/0021_add_delete_files_on_disk.py b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py deleted file mode 100644 index 5745d478..00000000 --- a/tubesync/sync/migrations/0021_add_delete_files_on_disk.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by pac - -from django.db import migrations, models - -class Migration(migrations.Migration): - - dependencies = [ - ('sync', '0020_auto_20231024_1825'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='delete_files_on_disk', - field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), - ), - ] \ No newline at end of file diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index a63c57e4..dff80638 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -300,11 +300,6 @@ class Source(models.Model): default=False, help_text=_('Delete media that is no longer on this playlist') ) - delete_files_on_disk = models.BooleanField( - _('delete files on disk'), - default=False, - help_text=_('Delete files on disk when they are removed from TubeSync') - ) source_resolution = models.CharField( _('source resolution'), max_length=8, diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index b9f88357..6600b718 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -224,7 +224,7 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) - if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb): + if instance.media_file or instance.thumb: # Delete all media files if it contains filename filepath = instance.media_file.path if instance.media_file else instance.thumb.path barefilepath, fileext = os.path.splitext(filepath) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index ac65d5cc..ba319c02 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -298,7 +298,7 @@ class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', + 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', From ad1d49a8359e9bfc70eeabeea72803c18e4a71bb Mon Sep 17 00:00:00 2001 From: administrator <7dn1yh5j@debauchez.fr> Date: Mon, 11 Dec 2023 03:25:17 +0100 Subject: [PATCH 05/13] Removed non-pertinent source option --- tubesync/sync/templates/sync/source.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 12b083e3..b9508389 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -122,11 +122,6 @@ Delete removed media Delete removed media
{% if source.delete_removed_media %}{% else %}{% endif %} - - - Delete files on disk - Delete files on disk
{% if source.delete_files_on_disk %}{% else %}{% endif %} - {% if source.delete_old_media and source.days_to_keep > 0 %} Delete old media From af94b37ee62c17adade34be2633dd2484a31455d Mon Sep 17 00:00:00 2001 From: administrator <7dn1yh5j@debauchez.fr> Date: Mon, 11 Dec 2023 03:38:14 +0100 Subject: [PATCH 06/13] Revert "Removed non-pertinent source option" This reverts commit ad1d49a8359e9bfc70eeabeea72803c18e4a71bb. t cherry-pick 46a43b9 Restore option --- tubesync/sync/templates/sync/source.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index b9508389..12b083e3 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -122,6 +122,11 @@ Delete removed media Delete removed media
{% if source.delete_removed_media %}{% else %}{% endif %} + + + Delete files on disk + Delete files on disk
{% if source.delete_files_on_disk %}{% else %}{% endif %} + {% if source.delete_old_media and source.days_to_keep > 0 %} Delete old media From ed07073cf431b7de57c289873867ea0cda9bdad0 Mon Sep 17 00:00:00 2001 From: administrator <7dn1yh5j@debauchez.fr> Date: Mon, 11 Dec 2023 03:38:37 +0100 Subject: [PATCH 07/13] Revert "Removed non-pertinent source option" This reverts commit 46ba2593a2d7d756676d6e57e0924326ee1bb74e. Restore option --- .../migrations/0021_add_delete_files_on_disk.py | 17 +++++++++++++++++ tubesync/sync/models.py | 5 +++++ tubesync/sync/signals.py | 2 +- tubesync/sync/views.py | 2 +- 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tubesync/sync/migrations/0021_add_delete_files_on_disk.py diff --git a/tubesync/sync/migrations/0021_add_delete_files_on_disk.py b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py new file mode 100644 index 00000000..5745d478 --- /dev/null +++ b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py @@ -0,0 +1,17 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0020_auto_20231024_1825'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index dff80638..a63c57e4 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -300,6 +300,11 @@ class Source(models.Model): default=False, help_text=_('Delete media that is no longer on this playlist') ) + delete_files_on_disk = models.BooleanField( + _('delete files on disk'), + default=False, + help_text=_('Delete files on disk when they are removed from TubeSync') + ) source_resolution = models.CharField( _('source resolution'), max_length=8, diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 6600b718..b9f88357 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -224,7 +224,7 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) - if instance.media_file or instance.thumb: + if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb): # Delete all media files if it contains filename filepath = instance.media_file.path if instance.media_file else instance.thumb.path barefilepath, fileext = os.path.splitext(filepath) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index ba319c02..ac65d5cc 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -298,7 +298,7 @@ class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'days_to_keep', 'source_resolution', + 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', From 4fdd172b05cd675517155ffb2da6ed532db1cff7 Mon Sep 17 00:00:00 2001 From: meeb Date: Tue, 12 Dec 2023 14:40:23 +1100 Subject: [PATCH 08/13] tidy up and pass a serialised list through to youtube.download_media, may help with #362 --- tubesync/sync/models.py | 26 +++++++++++--------------- tubesync/sync/youtube.py | 20 +++++++++----------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index e5c9221f..356e7792 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -18,7 +18,7 @@ from common.utils import clean_filename from .youtube import (get_media_info as get_youtube_media_info, download_media as download_youtube_media) from .utils import seconds_to_timestr, parse_media_format -from .matching import (get_best_combined_format, get_best_audio_format, +from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from .mediaservers import PlexMediaServer from .fields import CommaSepChoiceField @@ -107,7 +107,6 @@ class Source(models.Model): EXTENSION_MKV = 'mkv' EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) - # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py SPONSORBLOCK_CATEGORIES_CHOICES = ( ('sponsor', 'Sponsor'), @@ -121,15 +120,14 @@ class Source(models.Model): ) sponsorblock_categories = CommaSepChoiceField( - _(''), - possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, - all_choice="all", - allow_all=True, - all_label="(all options)", - default="all", - help_text=_("Select the sponsorblocks you want to enforce") - ) - + _(''), + possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, + all_choice='all', + allow_all=True, + all_label='(all options)', + default='all', + help_text=_('Select the sponsorblocks you want to enforce') + ) embed_metadata = models.BooleanField( _('embed metadata'), default=False, @@ -140,14 +138,12 @@ class Source(models.Model): default=False, help_text=_('Embed thumbnail into the file') ) - enable_sponsorblock = models.BooleanField( _('enable sponsorblock'), default=True, help_text=_('Use SponsorBlock?') ) - # Fontawesome icons used for the source on the front end ICONS = { SOURCE_TYPE_YOUTUBE_CHANNEL: '', @@ -1390,8 +1386,8 @@ class Media(models.Model): f'no valid format available') # Download the media with youtube-dl download_youtube_media(self.url, format_str, self.source.extension, - str(self.filepath), self.source.write_json, - self.source.sponsorblock_categories, self.source.embed_thumbnail, + str(self.filepath), self.source.write_json, + self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail, self.source.embed_metadata, self.source.enable_sponsorblock, self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs ) # Return the download paramaters diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 4ac6e838..bf3e9f56 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -1,5 +1,5 @@ ''' - Wrapper for the youtube-dl library. Used so if there are any library interface + Wrapper for the yt-dlp library. Used so if there are any library interface updates we only need to udpate them in one place. ''' @@ -64,9 +64,9 @@ def get_media_info(url): return response -def download_media(url, media_format, extension, output_file, info_json, - sponsor_categories="all", - embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, +def download_media(url, media_format, extension, output_file, info_json, + sponsor_categories=None, + embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, write_subtitles=False, auto_subtitles=False, sub_langs='en'): ''' Downloads a YouTube URL to a file on disk. @@ -74,7 +74,7 @@ def download_media(url, media_format, extension, output_file, info_json, def hook(event): filename = os.path.basename(event['filename']) - + if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: return None @@ -106,8 +106,8 @@ def download_media(url, media_format, extension, output_file, info_json, f'{total_size_str} in {elapsed_str}') else: log.warn(f'[youtube-dl] unknown event: {str(event)}') - hook.download_progress = 0 + hook.download_progress = 0 ytopts = { 'format': media_format, 'merge_output_format': extension, @@ -120,7 +120,8 @@ def download_media(url, media_format, extension, output_file, info_json, 'writeautomaticsub': auto_subtitles, 'subtitleslangs': sub_langs.split(','), } - + if not sponsor_categories: + sponsor_categories = [] sbopt = { 'key': 'SponsorBlock', 'categories': [sponsor_categories] @@ -130,7 +131,6 @@ def download_media(url, media_format, extension, output_file, info_json, 'add_chapters': True, 'add_metadata': True } - opts = get_yt_opts() if embed_thumbnail: ytopts['postprocessors'].append({'key': 'EmbedThumbnail'}) @@ -138,11 +138,9 @@ def download_media(url, media_format, extension, output_file, info_json, ffmdopt["add_metadata"] = True if skip_sponsors: ytopts['postprocessors'].append(sbopt) - ytopts['postprocessors'].append(ffmdopt) - opts.update(ytopts) - + with yt_dlp.YoutubeDL(opts) as y: try: return y.download([url]) From 7384c00713d2b4f4209d512ade2b0a2c263b31a6 Mon Sep 17 00:00:00 2001 From: meeb Date: Tue, 12 Dec 2023 16:28:26 +1100 Subject: [PATCH 09/13] fix typo in sponsorblock categories, remove dupe metadata flags, related to #362 --- tubesync/sync/youtube.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index bf3e9f56..cdf2647b 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -124,18 +124,16 @@ def download_media(url, media_format, extension, output_file, info_json, sponsor_categories = [] sbopt = { 'key': 'SponsorBlock', - 'categories': [sponsor_categories] + 'categories': sponsor_categories } ffmdopt = { 'key': 'FFmpegMetadata', - 'add_chapters': True, - 'add_metadata': True + 'add_chapters': embed_metadata, + 'add_metadata': embed_metadata } opts = get_yt_opts() if embed_thumbnail: ytopts['postprocessors'].append({'key': 'EmbedThumbnail'}) - if embed_metadata: - ffmdopt["add_metadata"] = True if skip_sponsors: ytopts['postprocessors'].append(sbopt) ytopts['postprocessors'].append(ffmdopt) From 1b581aa4ba619212628049fecabc36868c5aee2e Mon Sep 17 00:00:00 2001 From: Yottatron <54516274+ShaneBridges1234@users.noreply.github.com> Date: Mon, 1 Jan 2024 09:16:04 -0800 Subject: [PATCH 10/13] Update other-database-backends.md Correct table new in SQL for MariaDB column compression. --- docs/other-database-backends.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index 3ac61e42..4f90d3ab 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -84,7 +84,7 @@ With a lot of media files the `sync_media` table grows in size quickly. You can save space using column compression using the following steps while using MariaDB: 1. Stop tubesync - 2. Execute `ALTER TABLE sync_source MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync + 2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync 3. Start tunesync and confirm the connection still works. ## Docker Compose From 3573c1187f4e34a379850f818b0a44413d4723eb Mon Sep 17 00:00:00 2001 From: Dmitrii Savchenko Date: Tue, 16 Jan 2024 11:57:00 +0200 Subject: [PATCH 11/13] Optimized source page and dashboard loading --- tubesync/sync/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 0b808eb9..e2848545 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -59,7 +59,7 @@ class DashboardView(TemplateView): # Disk usage disk_usage = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).aggregate(Sum('downloaded_filesize')) + ).defer('metadata').aggregate(Sum('downloaded_filesize')) data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum'] if not data['disk_usage_bytes']: data['disk_usage_bytes'] = 0 @@ -71,11 +71,11 @@ class DashboardView(TemplateView): # Latest downloads data['latest_downloads'] = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).order_by('-download_date')[:10] + ).defer('metadata').order_by('-download_date')[:10] # Largest downloads data['largest_downloads'] = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).order_by('-downloaded_filesize')[:10] + ).defer('metadata').order_by('-downloaded_filesize')[:10] # UID and GID data['uid'] = os.getuid() data['gid'] = os.getgid() @@ -404,7 +404,7 @@ class SourceView(DetailView): error_message = get_error_message(error) setattr(error, 'error_message', error_message) data['errors'].append(error) - data['media'] = Media.objects.filter(source=self.object).order_by('-published') + data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata') return data From c6acd5378cf37fc7d505931c2dee32b61a0db368 Mon Sep 17 00:00:00 2001 From: meeb Date: Fri, 2 Feb 2024 05:51:20 +1100 Subject: [PATCH 12/13] move TIME_ZONE set by env var from local_settings to settings, resolves #462 --- tubesync/tubesync/local_settings.py.container | 3 --- tubesync/tubesync/settings.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index a7a07abc..a149ca71 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -25,9 +25,6 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) -TIME_ZONE = os.getenv('TZ', 'UTC') - - database_dict = {} database_connection_env = os.getenv('DATABASE_CONNECTION', '') if database_connection_env: diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index f1a333b7..671547b4 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,3 +1,4 @@ +import os from pathlib import Path @@ -96,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = os.getenv('TZ', 'UTC') USE_I18N = True USE_L10N = True USE_TZ = True From 246e2a31873ce6ec45258db37d804fbb3d494523 Mon Sep 17 00:00:00 2001 From: meeb Date: Mon, 26 Feb 2024 16:44:38 +1100 Subject: [PATCH 13/13] revert skip metadata if already skipped change, related to #293 --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index e819c8ae..b9f88357 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -172,7 +172,7 @@ def media_post_save(sender, instance, created, **kwargs): instance.save() post_save.connect(media_post_save, sender=Media) # If the media is missing metadata schedule it to be downloaded - if not instance.metadata and not instance.skip: + if not instance.metadata: log.info(f'Scheduling task to download metadata for: {instance.url}') verbose_name = _('Downloading metadata for "{}"') download_media_metadata( @@ -184,7 +184,7 @@ def media_post_save(sender, instance, created, **kwargs): # If the media is missing a thumbnail schedule it to be downloaded if not instance.thumb_file_exists: instance.thumb = None - if not instance.thumb and not instance.skip: + if not instance.thumb: thumbnail_url = instance.thumbnail if thumbnail_url: log.info(f'Scheduling task to download thumbnail for: {instance.name} '