diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md index 3ac61e42..4f90d3ab 100644 --- a/docs/other-database-backends.md +++ b/docs/other-database-backends.md @@ -84,7 +84,7 @@ With a lot of media files the `sync_media` table grows in size quickly. You can save space using column compression using the following steps while using MariaDB: 1. Stop tubesync - 2. Execute `ALTER TABLE sync_source MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync + 2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync 3. Start tunesync and confirm the connection still works. ## Docker Compose diff --git a/tubesync/sync/migrations/0021_add_delete_files_on_disk.py b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py new file mode 100644 index 00000000..5745d478 --- /dev/null +++ b/tubesync/sync/migrations/0021_add_delete_files_on_disk.py @@ -0,0 +1,17 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0020_auto_20231024_1825'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='delete_files_on_disk', + field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 55139dd7..ea974b0b 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -20,7 +20,7 @@ from .youtube import (get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) from .utils import seconds_to_timestr, parse_media_format -from .matching import (get_best_combined_format, get_best_audio_format, +from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from .mediaservers import PlexMediaServer from .fields import CommaSepChoiceField @@ -109,7 +109,6 @@ class Source(models.Model): EXTENSION_MKV = 'mkv' EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) - # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py SPONSORBLOCK_CATEGORIES_CHOICES = ( ('sponsor', 'Sponsor'), @@ -123,15 +122,14 @@ class Source(models.Model): ) sponsorblock_categories = CommaSepChoiceField( - _(''), - possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, - all_choice="all", - allow_all=True, - all_label="(all options)", - default="all", - help_text=_("Select the sponsorblocks you want to enforce") - ) - + _(''), + possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, + all_choice='all', + allow_all=True, + all_label='(all options)', + default='all', + help_text=_('Select the sponsorblocks you want to enforce') + ) embed_metadata = models.BooleanField( _('embed metadata'), default=False, @@ -142,14 +140,12 @@ class Source(models.Model): default=False, help_text=_('Embed thumbnail into the file') ) - enable_sponsorblock = models.BooleanField( _('enable sponsorblock'), default=True, help_text=_('Use SponsorBlock?') ) - # Fontawesome icons used for the source on the front end ICONS = { SOURCE_TYPE_YOUTUBE_CHANNEL: '', @@ -302,6 +298,11 @@ class Source(models.Model): default=False, help_text=_('Delete media that is no longer on this playlist') ) + delete_files_on_disk = models.BooleanField( + _('delete files on disk'), + default=False, + help_text=_('Delete files on disk when they are removed from TubeSync') + ) source_resolution = models.CharField( _('source resolution'), max_length=8, @@ -1405,8 +1406,8 @@ class Media(models.Model): f'no valid format available') # Download the media with youtube-dl download_youtube_media(self.url, format_str, self.source.extension, - str(self.filepath), self.source.write_json, - self.source.sponsorblock_categories, self.source.embed_thumbnail, + str(self.filepath), self.source.write_json, + self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail, self.source.embed_metadata, self.source.enable_sponsorblock, self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs ) # Return the download paramaters diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 6d2f1825..f019f2fc 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -1,4 +1,5 @@ import os +import glob from django.conf import settings from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.dispatch import receiver @@ -80,6 +81,7 @@ def source_pre_delete(sender, instance, **kwargs): media.delete() + @receiver(post_delete, sender=Source) def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted @@ -228,6 +230,16 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) + if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb): + # Delete all media files if it contains filename + filepath = instance.media_file.path if instance.media_file else instance.thumb.path + barefilepath, fileext = os.path.splitext(filepath) + # Get all files that start with the bare file path + all_related_files = glob.glob(f'{barefilepath}.*') + for file in all_related_files: + log.info(f'Deleting file for: {instance} path: {file}') + delete_file(file) + @receiver(post_delete, sender=Media) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index cd16326d..420de9be 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -344,6 +344,11 @@ def download_media_thumbnail(media_id, url): except Media.DoesNotExist: # Task triggered but the media no longer exists, do nothing return + if media.skip: + # Media was toggled to be skipped after the task was scheduled + log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but ' + f'it is now marked to be skipped, not downloading thumbnail') + return width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) i = get_remote_image(url) diff --git a/tubesync/sync/templates/sync/source-delete.html b/tubesync/sync/templates/sync/source-delete.html index bdc95208..ff4ef3b1 100644 --- a/tubesync/sync/templates/sync/source-delete.html +++ b/tubesync/sync/templates/sync/source-delete.html @@ -9,8 +9,8 @@

Are you sure you want to delete this source? Deleting a source is permanent. By default, deleting a source does not delete any saved media files. You can - tick the "also delete downloaded media" checkbox to also remove save - media when you delete the source. Deleting a source cannot be undone. + tick the "also delete downloaded media" checkbox to also remove directory {{ source.directory_path }} + when you delete the source. Deleting a source cannot be undone.

diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index c5812b28..12b083e3 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -122,6 +122,10 @@ Delete removed media Delete removed media
{% if source.delete_removed_media %}{% else %}{% endif %} + + + Delete files on disk + Delete files on disk
{% if source.delete_files_on_disk %}{% else %}{% endif %} {% if source.delete_old_media and source.days_to_keep > 0 %} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index a97b1ea2..ad02c018 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1,7 +1,9 @@ +import glob import os import json from base64 import b64decode import pathlib +import shutil import sys from django.conf import settings from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect @@ -59,7 +61,7 @@ class DashboardView(TemplateView): # Disk usage disk_usage = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).aggregate(Sum('downloaded_filesize')) + ).defer('metadata').aggregate(Sum('downloaded_filesize')) data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum'] if not data['disk_usage_bytes']: data['disk_usage_bytes'] = 0 @@ -71,11 +73,11 @@ class DashboardView(TemplateView): # Latest downloads data['latest_downloads'] = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).order_by('-download_date')[:10] + ).defer('metadata').order_by('-download_date')[:10] # Largest downloads data['largest_downloads'] = Media.objects.filter( downloaded=True, downloaded_filesize__isnull=False - ).order_by('-downloaded_filesize')[:10] + ).defer('metadata').order_by('-downloaded_filesize')[:10] # UID and GID data['uid'] = os.getuid() data['gid'] = os.getgid() @@ -298,6 +300,8 @@ class EditSourceMixin: 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images', + 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', + 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') @@ -404,7 +408,7 @@ class SourceView(DetailView): error_message = get_error_message(error) setattr(error, 'error_message', error_message) data['errors'].append(error) - data['media'] = Media.objects.filter(source=self.object).order_by('-published') + data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata') return data @@ -435,14 +439,13 @@ class DeleteSourceView(DeleteView, FormMixin): source = self.get_object() for media in Media.objects.filter(source=source): if media.media_file: - # Delete the media file - delete_file(media.media_file.path) - # Delete thumbnail copy if it exists - delete_file(media.thumbpath) - # Delete NFO file if it exists - delete_file(media.nfopath) - # Delete JSON file if it exists - delete_file(media.jsonpath) + file_path = media.media_file.path + matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*') + for file in matching_files: + delete_file(file) + directory_path = source.directory_path + if os.path.exists(directory_path): + shutil.rmtree(directory_path, True) return super().post(request, *args, **kwargs) def get_success_url(self): @@ -653,12 +656,13 @@ class MediaSkipView(FormView, SingleObjectMixin): delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) # If the media file exists on disk, delete it if self.object.media_file_exists: - delete_file(self.object.media_file.path) - self.object.media_file = None - # If the media has an associated thumbnail copied, also delete it - delete_file(self.object.thumbpath) - # If the media has an associated NFO file with it, also delete it - delete_file(self.object.nfopath) + # Delete all files which contains filename + filepath = self.object.media_file.path + barefilepath, fileext = os.path.splitext(filepath) + # Get all files that start with the bare file path + all_related_files = glob.glob(f'{barefilepath}.*') + for file in all_related_files: + delete_file(file) # Reset all download data self.object.metadata = None self.object.downloaded = False diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 1ef1fab1..78b130c5 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -1,5 +1,5 @@ ''' - Wrapper for the youtube-dl library. Used so if there are any library interface + Wrapper for the yt-dlp library. Used so if there are any library interface updates we only need to udpate them in one place. ''' @@ -93,9 +93,9 @@ def get_media_info(url): return response -def download_media(url, media_format, extension, output_file, info_json, - sponsor_categories="all", - embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, +def download_media(url, media_format, extension, output_file, info_json, + sponsor_categories=None, + embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, write_subtitles=False, auto_subtitles=False, sub_langs='en'): ''' Downloads a YouTube URL to a file on disk. @@ -103,7 +103,7 @@ def download_media(url, media_format, extension, output_file, info_json, def hook(event): filename = os.path.basename(event['filename']) - + if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: return None @@ -135,8 +135,8 @@ def download_media(url, media_format, extension, output_file, info_json, f'{total_size_str} in {elapsed_str}') else: log.warn(f'[youtube-dl] unknown event: {str(event)}') - hook.download_progress = 0 + hook.download_progress = 0 ytopts = { 'format': media_format, 'merge_output_format': extension, @@ -149,29 +149,25 @@ def download_media(url, media_format, extension, output_file, info_json, 'writeautomaticsub': auto_subtitles, 'subtitleslangs': sub_langs.split(','), } - + if not sponsor_categories: + sponsor_categories = [] sbopt = { 'key': 'SponsorBlock', - 'categories': [sponsor_categories] + 'categories': sponsor_categories } ffmdopt = { 'key': 'FFmpegMetadata', - 'add_chapters': True, - 'add_metadata': True + 'add_chapters': embed_metadata, + 'add_metadata': embed_metadata } - opts = get_yt_opts() if embed_thumbnail: ytopts['postprocessors'].append({'key': 'EmbedThumbnail'}) - if embed_metadata: - ffmdopt["add_metadata"] = True if skip_sponsors: ytopts['postprocessors'].append(sbopt) - ytopts['postprocessors'].append(ffmdopt) - opts.update(ytopts) - + with yt_dlp.YoutubeDL(opts) as y: try: return y.download([url]) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index a7a07abc..a149ca71 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -25,9 +25,6 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) -TIME_ZONE = os.getenv('TZ', 'UTC') - - database_dict = {} database_connection_env = os.getenv('DATABASE_CONNECTION', '') if database_connection_env: diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index f1a333b7..671547b4 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,3 +1,4 @@ +import os from pathlib import Path @@ -96,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = os.getenv('TZ', 'UTC') USE_I18N = True USE_L10N = True USE_TZ = True