From 4b60073ef06a76c87ed487cebd51f1e71e2190cb Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Dec 2024 10:38:22 -0500 Subject: [PATCH 01/34] Add multiple key sorting --- tubesync/sync/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index cf72462e..f49c9894 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -1,6 +1,7 @@ import os import re import math +from operator import itemgetter from pathlib import Path import requests from PIL import Image @@ -134,6 +135,15 @@ def seconds_to_timestr(seconds): return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) +def multi_key_sort(sort_dict, specs, use_reversed=False): + result = list(sort_dict) + for key, reverse in reversed(specs): + result = sorted(result, key=itemgetter(key), reverse=reverse) + if use_reversed: + return list(reversed(result)) + return result + + def parse_media_format(format_dict): ''' This parser primarily adapts the format dict returned by youtube-dl into a From ede9dff4e6b2cbbf8ca81eb6681c744d61fcbf89 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Dec 2024 10:56:19 -0500 Subject: [PATCH 02/34] Use new multiple key sorting --- tubesync/sync/matching.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index c453ff96..5d5b5dbd 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -5,6 +5,7 @@ ''' +from .utils import multi_key_sort from django.conf import settings @@ -49,6 +50,7 @@ def get_best_audio_format(media): ''' # Order all audio-only formats by bitrate audio_formats = [] + sort_keys = [('abr', True)] # key, reverse for fmt in media.iter_formats(): # If the format has a video stream, skip it if fmt['vcodec'] is not None: @@ -56,7 +58,7 @@ def get_best_audio_format(media): if not fmt['acodec']: continue audio_formats.append(fmt) - audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr']))) + audio_formats = list(multi_key_sort(audio_formats, sort_keys)) if not audio_formats: # Media has no audio formats at all return False, False @@ -86,6 +88,7 @@ def get_best_video_format(media): return False, False # Filter video-only formats by resolution that matches the source video_formats = [] + sort_keys = [('height', True), ('id', True)] # key, reverse for fmt in media.iter_formats(): # If the format has an audio stream, skip it if fmt['acodec'] is not None: @@ -109,7 +112,7 @@ def get_best_video_format(media): else: # Can't fallback return False, False - video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height']))) + video_formats = list(multi_key_sort(video_formats, sort_keys)) source_resolution = media.source.source_resolution.strip().upper() source_vcodec = media.source.source_vcodec if not video_formats: From 77247f7a7f01158252eb8415097338fc2bdbc6dc Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Dec 2024 13:33:00 -0500 Subject: [PATCH 03/34] Use reversed argument for multiple key sorting --- tubesync/sync/matching.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 5d5b5dbd..7fdc303a 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -50,7 +50,7 @@ def get_best_audio_format(media): ''' # Order all audio-only formats by bitrate audio_formats = [] - sort_keys = [('abr', True)] # key, reverse + sort_keys = [('abr', False)] # key, reverse for fmt in media.iter_formats(): # If the format has a video stream, skip it if fmt['vcodec'] is not None: @@ -58,7 +58,7 @@ def get_best_audio_format(media): if not fmt['acodec']: continue audio_formats.append(fmt) - audio_formats = list(multi_key_sort(audio_formats, sort_keys)) + audio_formats = multi_key_sort(audio_formats, sort_keys, True) if not audio_formats: # Media has no audio formats at all return False, False @@ -88,7 +88,7 @@ def get_best_video_format(media): return False, False # Filter video-only formats by resolution that matches the source video_formats = [] - sort_keys = [('height', True), ('id', True)] # key, reverse + sort_keys = [('height', False), ('id', False)] # key, reverse for fmt in media.iter_formats(): # If the format has an audio stream, skip it if fmt['acodec'] is not None: @@ -112,7 +112,7 @@ def get_best_video_format(media): else: # Can't fallback return False, False - video_formats = list(multi_key_sort(video_formats, sort_keys)) + video_formats = multi_key_sort(video_formats, sort_keys, True) source_resolution = media.source.source_resolution.strip().upper() source_vcodec = media.source.source_vcodec if not video_formats: From 526c6a97b4f6748f68563a4b4f7449125eb0ec92 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 2 Dec 2024 23:09:10 -0500 Subject: [PATCH 04/34] Match against height instead of format --- tubesync/sync/matching.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 7fdc303a..75009dff 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -22,7 +22,7 @@ def get_best_combined_format(media): ''' for fmt in media.iter_formats(): # Check height matches - if media.source.source_resolution.strip().upper() != fmt['format']: + if media.source.source_resolution_height != fmt['height']: continue # Check the video codec matches if media.source.source_vcodec != fmt['vcodec']: @@ -97,6 +97,8 @@ def get_best_video_format(media): continue if media.source.source_resolution.strip().upper() == fmt['format']: video_formats.append(fmt) + elif media.source.source_resolution_height == fmt['height']: + video_formats.append(fmt) # Check we matched some streams if not video_formats: # No streams match the requested resolution, see if we can fallback From ed98f56946301e6138691ede107c7d4234695736 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 4 Dec 2024 16:36:52 -0500 Subject: [PATCH 05/34] Adjust VP09 vcodec to match VP09 source vcodec --- tubesync/sync/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index f49c9894..3dfcba2d 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -158,6 +158,8 @@ def parse_media_format(format_dict): vcodec = None if vcodec == 'NONE': vcodec = None + if vcodec == 'VP09': + vcodec = 'VP9' acodec_full = format_dict.get('acodec', '') acodec_parts = acodec_full.split('.') if len(acodec_parts) > 0: From 6d7d3483a43374d46ccb94a50967ab2415f22194 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 4 Dec 2024 16:52:10 -0500 Subject: [PATCH 06/34] Use the audio format list from youtube-dl --- tubesync/sync/matching.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 75009dff..6a72972d 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -48,9 +48,8 @@ def get_best_audio_format(media): Finds the best match for the source required audio format. If the source has a 'fallback' of fail this can return no match. ''' - # Order all audio-only formats by bitrate + # Reverse order all audio-only formats audio_formats = [] - sort_keys = [('abr', False)] # key, reverse for fmt in media.iter_formats(): # If the format has a video stream, skip it if fmt['vcodec'] is not None: @@ -58,18 +57,18 @@ def get_best_audio_format(media): if not fmt['acodec']: continue audio_formats.append(fmt) - audio_formats = multi_key_sort(audio_formats, sort_keys, True) + audio_formats = list(reversed(audio_formats)) if not audio_formats: # Media has no audio formats at all return False, False - # Find the highest bitrate audio format with a matching codec + # Find the first audio format with a matching codec for fmt in audio_formats: if media.source.source_acodec == fmt['acodec']: # Matched! return True, fmt['id'] # No codecs matched if media.source.can_fallback: - # Can fallback, find the next highest bitrate non-matching codec + # Can fallback, find the next non-matching codec return False, audio_formats[0]['id'] else: # Can't fallback From 09044aa95d4e8cf5c5c4926eb08da3de2e7fe8a4 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 4 Dec 2024 17:03:18 -0500 Subject: [PATCH 07/34] Return for empty format lists first --- tubesync/sync/matching.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 6a72972d..982af040 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -57,10 +57,10 @@ def get_best_audio_format(media): if not fmt['acodec']: continue audio_formats.append(fmt) - audio_formats = list(reversed(audio_formats)) if not audio_formats: # Media has no audio formats at all return False, False + audio_formats = list(reversed(audio_formats)) # Find the first audio format with a matching codec for fmt in audio_formats: if media.source.source_acodec == fmt['acodec']: @@ -113,12 +113,12 @@ def get_best_video_format(media): else: # Can't fallback return False, False - video_formats = multi_key_sort(video_formats, sort_keys, True) - source_resolution = media.source.source_resolution.strip().upper() - source_vcodec = media.source.source_vcodec if not video_formats: # Still no matches return False, False + video_formats = multi_key_sort(video_formats, sort_keys, True) + source_resolution = media.source.source_resolution.strip().upper() + source_vcodec = media.source.source_vcodec exact_match, best_match = None, None # Of our filtered video formats, check for resolution + codec + hdr + fps match if media.source.prefer_60fps and media.source.prefer_hdr: From b1a4c8bbaf244591e3b8af781fa61a13c5882bdf Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 4 Dec 2024 17:08:37 -0500 Subject: [PATCH 08/34] Include streams with a blank format_note and the proper height --- tubesync/sync/matching.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index 982af040..0b1f8d34 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -87,7 +87,7 @@ def get_best_video_format(media): return False, False # Filter video-only formats by resolution that matches the source video_formats = [] - sort_keys = [('height', False), ('id', False)] # key, reverse + sort_keys = [('height', False), ('vcodec', True), ('vbr', False)] # key, reverse for fmt in media.iter_formats(): # If the format has an audio stream, skip it if fmt['acodec'] is not None: @@ -120,6 +120,10 @@ def get_best_video_format(media): source_resolution = media.source.source_resolution.strip().upper() source_vcodec = media.source.source_vcodec exact_match, best_match = None, None + for fmt in video_formats: + # format_note was blank, match height instead + if '' == fmt['format'] and fmt['height'] == media.source.source_resolution_height: + fmt['format'] = source_resolution # Of our filtered video formats, check for resolution + codec + hdr + fps match if media.source.prefer_60fps and media.source.prefer_hdr: for fmt in video_formats: From f570fb57b9bec1b9c181b4d7386e61169e3d29d1 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 5 Dec 2024 07:57:24 -0500 Subject: [PATCH 09/34] Adjust to correct 60 fps case --- tubesync/sync/matching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index c453ff96..93fcdae3 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -331,7 +331,7 @@ def get_best_video_format(media): for fmt in video_formats: # Check for a codec, hdr and fps match but drop the resolution if (source_vcodec == fmt['vcodec'] and - not fmt['is_hdr'] and fmt['is_60fps']): + not fmt['is_hdr'] and not fmt['is_60fps']): # Close match exact_match, best_match = False, fmt break From 6fdcc5ffa5494fe25f53aef4e96ca6f7a319432d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 5 Dec 2024 17:56:53 -0500 Subject: [PATCH 10/34] Handle STR0NNN in a generic way The function should return 'VP9' for: - vp9.1.2.3 - vp09.4.5.6 As well as other variations. It also works for avc01 or av01, or any others that fit the pattern. --- tubesync/sync/utils.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index cf72462e..168cccf0 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -134,6 +134,21 @@ def seconds_to_timestr(seconds): return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) +def normalize_codec(codec_str): + result = str(codec_str) + parts = result.split('.') + if len(parts) > 0: + result = parts[0].strip() + else: + return None + if 'NONE' == result: + return None + if str(0) in result: + prefix = result.rstrip('0123456789') + result = prefix + str(int(result[len(prefix):])) + return result.upper() + + def parse_media_format(format_dict): ''' This parser primarily adapts the format dict returned by youtube-dl into a @@ -141,21 +156,9 @@ def parse_media_format(format_dict): any internals, update it here. ''' vcodec_full = format_dict.get('vcodec', '') - vcodec_parts = vcodec_full.split('.') - if len(vcodec_parts) > 0: - vcodec = vcodec_parts[0].strip().upper() - else: - vcodec = None - if vcodec == 'NONE': - vcodec = None + vcodec = normalize_codec(vcodec_full) acodec_full = format_dict.get('acodec', '') - acodec_parts = acodec_full.split('.') - if len(acodec_parts) > 0: - acodec = acodec_parts[0].strip().upper() - else: - acodec = None - if acodec == 'NONE': - acodec = None + acodec = normalize_codec(acodec_full) try: fps = int(format_dict.get('fps', 0)) except (ValueError, TypeError): From fb663d508fd0efdf6b6879b02a5c1fe0116c4d4b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 5 Dec 2024 18:27:46 -0500 Subject: [PATCH 11/34] Uppercase the codec string first, not last --- tubesync/sync/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 168cccf0..33ea678c 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -135,7 +135,7 @@ def seconds_to_timestr(seconds): def normalize_codec(codec_str): - result = str(codec_str) + result = str(codec_str).upper() parts = result.split('.') if len(parts) > 0: result = parts[0].strip() @@ -146,7 +146,7 @@ def normalize_codec(codec_str): if str(0) in result: prefix = result.rstrip('0123456789') result = prefix + str(int(result[len(prefix):])) - return result.upper() + return result def parse_media_format(format_dict): From 0fafca037c0c7db26bcdc3569793b7d8d6a40760 Mon Sep 17 00:00:00 2001 From: James W Lane Date: Mon, 9 Dec 2024 09:33:28 -0600 Subject: [PATCH 12/34] updated tubesync type directory path --- .gitignore | 3 ++ README.md | 14 ++++++++ tubesync/sync/models.py | 31 +++++++++++++++-- tubesync/sync/tests.py | 76 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index cb14cfd6..cf63cfba 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ dmypy.json Pipfile.lock .vscode/launch.json + +# Ignore Jetbrains IDE files +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index b85b2b4a..f1fdb6da 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,21 @@ useful if you are manually installing TubeSync in some other environment. These | HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | | DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | | VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_DIRECTORY_MODE | Controls how downloaded files are organized. | default | +# TubeSync Directory Mode + +Controls how downloaded files are organized. + +Values: +- default: Audio files go to `audio`, video files to `video`. +- flat: All files are placed in the root of DOWNLOAD_DIR. +- custom:,: Allows custom prefixes for audio and video directories under DOWNLOAD_DIR. + +Example: +``` +TUBESYNC_DIRECTORY_MODE=custom:music,shows +``` # Manual, non-containerised, installation diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2f116356..62cd8e9a 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -514,10 +514,35 @@ class Source(models.Model): @property def type_directory_path(self): - if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + # Get the directory mode from the environment + directory_mode = os.getenv("TUBESYNC_DIRECTORY_MODE", "default") + + # Default behavior + if directory_mode == "default": + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + else: + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + + # Flat mode - Place all files in the root + elif directory_mode == "flat": + return Path(settings.DOWNLOAD_DIR) / self.directory + + # Custom mode - Parse prefixes for audio and video + elif directory_mode.startswith("custom:"): + try: + audio_prefix, video_prefix = directory_mode.split(":")[1].split(",") + except ValueError: + raise ValueError("Invalid format for TUBESYNC_DIRECTORY_MODE=custom. Expected 'custom:audio_prefix,video_prefix'.") + + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: + return Path(settings.DOWNLOAD_DIR) / audio_prefix / self.directory + else: + return Path(settings.DOWNLOAD_DIR) / video_prefix / self.directory + + # Fallback for invalid modes else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + raise ValueError(f"Unsupported TUBESYNC_DIRECTORY_MODE: {directory_mode}") def make_directory(self): return os.makedirs(self.directory_path, exist_ok=True) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 72411894..0029a0e9 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -6,7 +6,9 @@ import logging +import os from datetime import datetime, timedelta +from pathlib import Path from urllib.parse import urlsplit from xml.etree import ElementTree from django.conf import settings @@ -1714,3 +1716,77 @@ class TasksTestCase(TestCase): self.assertEqual(src1.media_source.all().count(), 3) self.assertEqual(src2.media_source.all().count(), 2) self.assertEqual(Media.objects.filter(pk=m22.pk).exists(), False) + +class TypeDirectoryPathTestCase(TestCase): + def setUp(self): + # Mock settings for testing + self.audio_dir = Path("/mock/audio/dir") + self.video_dir = Path("/mock/video/dir") + self.download_dir = Path("/mock/download/dir") + settings.DOWNLOAD_AUDIO_DIR = self.audio_dir + settings.DOWNLOAD_VIDEO_DIR = self.video_dir + settings.DOWNLOAD_DIR = self.download_dir + + # Create a source object for testing + self.source = Source( + directory="test_directory", + source_resolution=Source.SOURCE_RESOLUTION_AUDIO, + ) + + def test_default_mode_audio(self): + """ + Test default mode for audio resolution. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "default" + expected_path = self.audio_dir / self.source.directory + self.assertEqual(self.source.type_directory_path, expected_path) + + def test_default_mode_video(self): + """ + Test default mode for video resolution. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "default" + self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P + expected_path = self.video_dir / self.source.directory + self.assertEqual(self.source.type_directory_path, expected_path) + + def test_flat_mode(self): + """ + Test flat mode places files in the root download directory. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "flat" + expected_path = self.download_dir / self.source.directory + self.assertEqual(self.source.type_directory_path, expected_path) + + def test_custom_mode_audio(self): + """ + Test custom mode with prefixes for audio. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:audio_prefix,video_prefix" + expected_path = self.download_dir / "audio_prefix" / self.source.directory + self.assertEqual(self.source.type_directory_path, expected_path) + + def test_custom_mode_video(self): + """ + Test custom mode with prefixes for video. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:audio_prefix,video_prefix" + self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P + expected_path = self.download_dir / "video_prefix" / self.source.directory + self.assertEqual(self.source.type_directory_path, expected_path) + + def test_custom_mode_invalid_format(self): + """ + Test custom mode with an invalid format. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:only_audio_prefix" + with self.assertRaises(ValueError): + _ = self.source.type_directory_path + + def test_invalid_mode(self): + """ + Test unsupported directory mode raises an error. + """ + os.environ["TUBESYNC_DIRECTORY_MODE"] = "unsupported_mode" + with self.assertRaises(ValueError): + _ = self.source.type_directory_path \ No newline at end of file From 279a01f9bc541b9af503468a43b88db830802af9 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 14:06:35 -0500 Subject: [PATCH 13/34] directory mode on settings --- tubesync/tubesync/settings.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index d67d577a..c43bf15c 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -112,6 +112,15 @@ DOWNLOAD_VIDEO_DIR = 'video' DOWNLOAD_AUDIO_DIR = 'audio' SASS_PROCESSOR_ROOT = STATIC_ROOT +directory_mode = os.getenv("TUBESYNC_DIRECTORY_MODE", "default") +if directory_mode == 'flat': + DOWNLOAD_VIDEO_DIR = '.' + DOWNLOAD_AUDIO_DIR = '.' +elif directory_mode.startswith("custom:"): + custom_value = directory_mode.split(":")[1] + if str(',') in custom_value: + DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',') + ROBOTS = ''' User-agent: * From a02fd5bde925662a7a2cf88ea9b30f016df44722 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 14:21:55 -0500 Subject: [PATCH 14/34] maxsplit and quoting --- tubesync/tubesync/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index c43bf15c..27af1d04 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -112,14 +112,14 @@ DOWNLOAD_VIDEO_DIR = 'video' DOWNLOAD_AUDIO_DIR = 'audio' SASS_PROCESSOR_ROOT = STATIC_ROOT -directory_mode = os.getenv("TUBESYNC_DIRECTORY_MODE", "default") +directory_mode = os.getenv('TUBESYNC_DIRECTORY_MODE', 'default') if directory_mode == 'flat': DOWNLOAD_VIDEO_DIR = '.' DOWNLOAD_AUDIO_DIR = '.' -elif directory_mode.startswith("custom:"): - custom_value = directory_mode.split(":")[1] +elif directory_mode.startswith('custom:'): + custom_value = directory_mode.split(':', maxsplit=1)[1] if str(',') in custom_value: - DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',') + DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',', maxsplit=1) ROBOTS = ''' From 233d384e25ddd44d5c8261d01a7d1d90ccd48729 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 14:56:41 -0500 Subject: [PATCH 15/34] Removed unnecessary str --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 27af1d04..5eb7a8dc 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -118,7 +118,7 @@ if directory_mode == 'flat': DOWNLOAD_AUDIO_DIR = '.' elif directory_mode.startswith('custom:'): custom_value = directory_mode.split(':', maxsplit=1)[1] - if str(',') in custom_value: + if ',' in custom_value: DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',', maxsplit=1) From 2a63e3a360d262b20239abf36dcf621a1048d444 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 15:26:12 -0500 Subject: [PATCH 16/34] Reset tubesync/sync/models.py (#16) --- tubesync/sync/models.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 62cd8e9a..2f116356 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -514,35 +514,10 @@ class Source(models.Model): @property def type_directory_path(self): - # Get the directory mode from the environment - directory_mode = os.getenv("TUBESYNC_DIRECTORY_MODE", "default") - - # Default behavior - if directory_mode == "default": - if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory - else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory - - # Flat mode - Place all files in the root - elif directory_mode == "flat": - return Path(settings.DOWNLOAD_DIR) / self.directory - - # Custom mode - Parse prefixes for audio and video - elif directory_mode.startswith("custom:"): - try: - audio_prefix, video_prefix = directory_mode.split(":")[1].split(",") - except ValueError: - raise ValueError("Invalid format for TUBESYNC_DIRECTORY_MODE=custom. Expected 'custom:audio_prefix,video_prefix'.") - - if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return Path(settings.DOWNLOAD_DIR) / audio_prefix / self.directory - else: - return Path(settings.DOWNLOAD_DIR) / video_prefix / self.directory - - # Fallback for invalid modes + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory else: - raise ValueError(f"Unsupported TUBESYNC_DIRECTORY_MODE: {directory_mode}") + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory def make_directory(self): return os.makedirs(self.directory_path, exist_ok=True) From a487e64550282d7d704824ace1fd3af87aeb9b5a Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 15:40:16 -0500 Subject: [PATCH 17/34] Raise an error for an unsupported value --- tubesync/tubesync/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 5eb7a8dc..1fd8499c 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -120,6 +120,8 @@ elif directory_mode.startswith('custom:'): custom_value = directory_mode.split(':', maxsplit=1)[1] if ',' in custom_value: DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',', maxsplit=1) +elif directory_mode not in ('', 'default'): + raise ValueError(f"Unsupported TUBESYNC_DIRECTORY_MODE: {directory_mode}") ROBOTS = ''' From d33be55b51857afe8d97ee9b22ea784d0c9130e9 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 15:44:32 -0500 Subject: [PATCH 18/34] Raise an error for an invalid custom value --- tubesync/tubesync/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 1fd8499c..220f93d4 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -120,6 +120,8 @@ elif directory_mode.startswith('custom:'): custom_value = directory_mode.split(':', maxsplit=1)[1] if ',' in custom_value: DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',', maxsplit=1) + else: + raise ValueError("Invalid format for TUBESYNC_DIRECTORY_MODE=custom. Expected 'custom:audio_prefix,video_prefix'.") elif directory_mode not in ('', 'default'): raise ValueError(f"Unsupported TUBESYNC_DIRECTORY_MODE: {directory_mode}") From 049c6f37641fc5e6e628b705c373d5610848a84e Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 9 Dec 2024 18:05:45 -0500 Subject: [PATCH 19/34] Accept '-' in sub_langs --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2f116356..88fd0b12 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -422,7 +422,7 @@ class Source(models.Model): help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), validators=[ RegexValidator( - regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$", + regex=r"^(\-?[\_\.a-zA-Z-]+,)*(\-?[\_\.a-zA-Z-]+){1}$", message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') ) ] From aa3bc7e6af4b7fa8839cff4d01ba86aff3d42116 Mon Sep 17 00:00:00 2001 From: James W Lane Date: Mon, 9 Dec 2024 22:23:32 -0600 Subject: [PATCH 20/34] fixed default test --- tubesync/sync/tests.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 0029a0e9..1975ba08 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1719,15 +1719,9 @@ class TasksTestCase(TestCase): class TypeDirectoryPathTestCase(TestCase): def setUp(self): - # Mock settings for testing - self.audio_dir = Path("/mock/audio/dir") - self.video_dir = Path("/mock/video/dir") - self.download_dir = Path("/mock/download/dir") - settings.DOWNLOAD_AUDIO_DIR = self.audio_dir - settings.DOWNLOAD_VIDEO_DIR = self.video_dir - settings.DOWNLOAD_DIR = self.download_dir - - # Create a source object for testing + self.audio_dir = Path(settings.DOWNLOAD_AUDIO_DIR) + self.video_dir = Path(settings.DOWNLOAD_VIDEO_DIR) + self.download_dir = Path(settings.DOWNLOAD_ROOT) self.source = Source( directory="test_directory", source_resolution=Source.SOURCE_RESOLUTION_AUDIO, From a4dc416ce8e2cbb123fdbe1db0889756030e5683 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 10 Dec 2024 00:05:58 -0500 Subject: [PATCH 21/34] Simplify directory_path This looked to be identical code to what `filename` used. Remove the duplicated code by using the function instead. --- tubesync/sync/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2f116356..d9f04ff1 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1291,10 +1291,7 @@ class Media(models.Model): @property def directory_path(self): - # Otherwise, create a suitable filename from the source media_format - media_format = str(self.source.media_format) - media_details = self.format_dict - dirname = self.source.directory_path / media_format.format(**media_details) + dirname = self.source.directory_path / self.filename return os.path.dirname(str(dirname)) @property From 63d0f4f15a563f71bd66dff55f41fad2e700276f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 10 Dec 2024 02:02:30 -0500 Subject: [PATCH 22/34] Simplify regex for sub_langs --- tubesync/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 88fd0b12..4f9de7f7 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -422,7 +422,7 @@ class Source(models.Model): help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), validators=[ RegexValidator( - regex=r"^(\-?[\_\.a-zA-Z-]+,)*(\-?[\_\.a-zA-Z-]+){1}$", + regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') ) ] From 26cb02be079fafc6e466a284ed3a8d9f4546d9ea Mon Sep 17 00:00:00 2001 From: meeb Date: Tue, 10 Dec 2024 22:30:09 +1100 Subject: [PATCH 23/34] bump yt-dlp and ffmpeg, resolves #575 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce4b358b..c5299147 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,10 @@ ARG SHA256_S6_AMD64="ad982a801bd72757c7b1b53539a146cf715e640b4d8f0a6a671a3d1b560 ARG SHA256_S6_ARM64="868973e98210257bba725ff5b17aa092008c9a8e5174499e38ba611a8fc7e473" ARG SHA256_S6_NOARCH="4b0c0907e6762814c31850e0e6c6762c385571d4656eb8725852b0b1586713b6" -ARG FFMPEG_DATE="autobuild-2024-11-22-14-18" -ARG FFMPEG_VERSION="117857-g2d077f9acd" -ARG SHA256_FFMPEG_AMD64="427ff38cf1e28521aac4fa7931444f6f2ff6f097f2d4a315c6f92ef1d7f90db8" -ARG SHA256_FFMPEG_ARM64="f7ed3a50b651447477aa2637bf8da6010d8f27f8804a5d0e77b00a1d3725b27a" +ARG FFMPEG_DATE="autobuild-2024-12-09-14-16" +ARG FFMPEG_VERSION="118034-gd21134313f" +ARG SHA256_FFMPEG_AMD64="cd50122fb0939e913585282347a8f95074c2d5477ceb059cd90aca551f14e9ea" +ARG SHA256_FFMPEG_ARM64="33b4edebf9c23701473ba8db696b26072bb9b9c05fc4a156e115f94e44d361e0" ENV S6_VERSION="${S6_VERSION}" \ FFMPEG_DATE="${FFMPEG_DATE}" \ From bd72513d69a11eac5600cec920314e90f3fc2fb5 Mon Sep 17 00:00:00 2001 From: meeb Date: Tue, 10 Dec 2024 22:36:45 +1100 Subject: [PATCH 24/34] bump s6 to v3.2.0.2 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5299147..4941821b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM debian:bookworm-slim ARG TARGETARCH ARG TARGETPLATFORM -ARG S6_VERSION="3.2.0.0" -ARG SHA256_S6_AMD64="ad982a801bd72757c7b1b53539a146cf715e640b4d8f0a6a671a3d1b560fe1e2" -ARG SHA256_S6_ARM64="868973e98210257bba725ff5b17aa092008c9a8e5174499e38ba611a8fc7e473" -ARG SHA256_S6_NOARCH="4b0c0907e6762814c31850e0e6c6762c385571d4656eb8725852b0b1586713b6" +ARG S6_VERSION="3.2.0.2" +ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478" +ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785" +ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae" ARG FFMPEG_DATE="autobuild-2024-12-09-14-16" ARG FFMPEG_VERSION="118034-gd21134313f" From ff8d00f5bd8a0af5ccc079bebe09b46a7175478b Mon Sep 17 00:00:00 2001 From: James W Lane Date: Tue, 10 Dec 2024 20:04:42 -0600 Subject: [PATCH 25/34] updating code per comments --- README.md | 14 +++----- tubesync/sync/tests.py | 64 +++++++---------------------------- tubesync/tubesync/settings.py | 13 ++----- 3 files changed, 18 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index f1fdb6da..9a565c29 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ There are a number of other environment variables you can set. These are, mostly useful if you are manually installing TubeSync in some other environment. These are: | Name | What | Example | -| --------------------------- | ------------------------------------------------------------ | ------------------------------------ | +| --------------------------- | ------------------------------------------------------------ |--------------------------------------| | DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | | DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | | TUBESYNC_DEBUG | Enable debugging | True | @@ -376,21 +376,15 @@ useful if you are manually installing TubeSync in some other environment. These | HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | | DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | | VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_DIRECTORY_MODE | Controls how downloaded files are organized. | default | +| TUBESYNC_DIRECTORY_PREFIX | Controls how downloaded files are organized. | true | # TubeSync Directory Mode Controls how downloaded files are organized. Values: -- default: Audio files go to `audio`, video files to `video`. -- flat: All files are placed in the root of DOWNLOAD_DIR. -- custom:,: Allows custom prefixes for audio and video directories under DOWNLOAD_DIR. - -Example: -``` -TUBESYNC_DIRECTORY_MODE=custom:music,shows -``` +- true: Audio files go to `audio`, video files to `video`. +- false: All files are placed in the root of DOWNLOAD_DIR. # Manual, non-containerised, installation diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 1975ba08..86a39366 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1719,68 +1719,28 @@ class TasksTestCase(TestCase): class TypeDirectoryPathTestCase(TestCase): def setUp(self): - self.audio_dir = Path(settings.DOWNLOAD_AUDIO_DIR) - self.video_dir = Path(settings.DOWNLOAD_VIDEO_DIR) - self.download_dir = Path(settings.DOWNLOAD_ROOT) self.source = Source( directory="test_directory", source_resolution=Source.SOURCE_RESOLUTION_AUDIO, ) - def test_default_mode_audio(self): + def test_directory_prefix_default(self): """ - Test default mode for audio resolution. + Test that default directory prefix exist. """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "default" - expected_path = self.audio_dir / self.source.directory - self.assertEqual(self.source.type_directory_path, expected_path) + os.environ['TUBESYNC_DIRECTORY_PREFIX'] = '' + self.assertEqual(self.source.type_directory_path, Path(settings.DOWNLOAD_AUDIO_DIR) / 'test_directory') - def test_default_mode_video(self): + def test_directory_prefix_true(self): """ - Test default mode for video resolution. + Test that when TUBESYNC_DIRECTORY_PREFIX is set to true the directory prefix exist. """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "default" - self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P - expected_path = self.video_dir / self.source.directory - self.assertEqual(self.source.type_directory_path, expected_path) + os.environ['TUBESYNC_DIRECTORY_PREFIX'] = 'true' + self.assertEqual(self.source.type_directory_path, Path(settings.DOWNLOAD_AUDIO_DIR) / 'test_directory') - def test_flat_mode(self): + def test_directory_prefix_false(self): """ - Test flat mode places files in the root download directory. + Test that when TUBESYNC_DIRECTORY_PREFIX is set to false the directory prefix does not exist. """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "flat" - expected_path = self.download_dir / self.source.directory - self.assertEqual(self.source.type_directory_path, expected_path) - - def test_custom_mode_audio(self): - """ - Test custom mode with prefixes for audio. - """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:audio_prefix,video_prefix" - expected_path = self.download_dir / "audio_prefix" / self.source.directory - self.assertEqual(self.source.type_directory_path, expected_path) - - def test_custom_mode_video(self): - """ - Test custom mode with prefixes for video. - """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:audio_prefix,video_prefix" - self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P - expected_path = self.download_dir / "video_prefix" / self.source.directory - self.assertEqual(self.source.type_directory_path, expected_path) - - def test_custom_mode_invalid_format(self): - """ - Test custom mode with an invalid format. - """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "custom:only_audio_prefix" - with self.assertRaises(ValueError): - _ = self.source.type_directory_path - - def test_invalid_mode(self): - """ - Test unsupported directory mode raises an error. - """ - os.environ["TUBESYNC_DIRECTORY_MODE"] = "unsupported_mode" - with self.assertRaises(ValueError): - _ = self.source.type_directory_path \ No newline at end of file + os.environ['TUBESYNC_DIRECTORY_PREFIX'] = 'false' + self.assertEqual(self.source.type_directory_path, Path('.') / 'test_directory') \ No newline at end of file diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 220f93d4..7bb4b2a2 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -112,19 +112,10 @@ DOWNLOAD_VIDEO_DIR = 'video' DOWNLOAD_AUDIO_DIR = 'audio' SASS_PROCESSOR_ROOT = STATIC_ROOT -directory_mode = os.getenv('TUBESYNC_DIRECTORY_MODE', 'default') -if directory_mode == 'flat': +directory_prefix = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'true') +if directory_prefix == 'false': DOWNLOAD_VIDEO_DIR = '.' DOWNLOAD_AUDIO_DIR = '.' -elif directory_mode.startswith('custom:'): - custom_value = directory_mode.split(':', maxsplit=1)[1] - if ',' in custom_value: - DOWNLOAD_AUDIO_DIR, DOWNLOAD_VIDEO_DIR = custom_value.split(',', maxsplit=1) - else: - raise ValueError("Invalid format for TUBESYNC_DIRECTORY_MODE=custom. Expected 'custom:audio_prefix,video_prefix'.") -elif directory_mode not in ('', 'default'): - raise ValueError(f"Unsupported TUBESYNC_DIRECTORY_MODE: {directory_mode}") - ROBOTS = ''' User-agent: * From d9c56db62741c53f458e232fc88e9fda10ab3841 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 11 Dec 2024 05:35:09 -0500 Subject: [PATCH 26/34] Disable checking against days to keep This isn't the logic we should be using. Just stop using it until the function is comparing the proper dates. --- tubesync/sync/filtering.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index 351b3f83..fb0a6b16 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -23,8 +23,8 @@ def filter_media(instance: Media): skip = True # Check if older than source_cutoff - if filter_source_cutoff(instance): - skip = True + #if filter_source_cutoff(instance): + # skip = True # Check if we have filter_text and filter text matches if filter_filter_text(instance): @@ -128,6 +128,7 @@ def filter_max_cap(instance: Media): # If the source has a cut-off, check the upload date is within the allowed delta +# TODO: days to keep should be compared to downloaded date, not published def filter_source_cutoff(instance: Media): if instance.source.delete_old_media and instance.source.days_to_keep > 0: if not isinstance(instance.published, datetime): From bb73cc6c97de444215d988b0d54b32158fe44de9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 11 Dec 2024 06:43:26 -0500 Subject: [PATCH 27/34] Add days_to_keep_date --- tubesync/sync/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 4f9de7f7..b769208c 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -460,6 +460,14 @@ class Source(models.Model): else: return False + @property + def days_to_keep_date(self): + delta = self.days_to_keep + if delta > 0: + return timezone.now() - timedelta(days=delta) + else: + return False + @property def extension(self): ''' From 5f28b8be661849071a54190b84062d562bb20abd Mon Sep 17 00:00:00 2001 From: meeb Date: Wed, 11 Dec 2024 22:56:21 +1100 Subject: [PATCH 28/34] tweaks related to #572 --- tubesync/sync/models.py | 9 ++-- tubesync/sync/tests.py | 51 ++++++++----------- tubesync/tubesync/local_settings.py.container | 3 ++ tubesync/tubesync/settings.py | 8 +++ 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 4f9de7f7..e23c91d5 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -514,10 +514,13 @@ class Source(models.Model): @property def type_directory_path(self): - if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + else: + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + return Path(self.directory) def make_directory(self): return os.makedirs(self.directory_path, exist_ok=True) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 86a39366..6aa0ccb6 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -628,6 +628,25 @@ class FilepathTestCase(TestCase): ('no-fancy-stuff-title_test_720p-720x1280-opus' '-vp9-30fps-hdr.mkv')) + def test_directory_prefix(self): + # Confirm the setting exists and is valid + self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX')) + self.assertTrue(isinstance(settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX, bool)) + # Test the default behavior for "True", forced "audio" or "video" parent directories for sources + settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True + self.source.source_resolution = Source.SOURCE_RESOLUTION_AUDIO + test_audio_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_audio_prefix_path.parts[-2], 'audio') + self.assertEqual(test_audio_prefix_path.parts[-1], 'testdirectory') + self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P + test_video_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_video_prefix_path.parts[-2], 'video') + self.assertEqual(test_video_prefix_path.parts[-1], 'testdirectory') + # Test the default behavior for "False", no parent directories for sources + settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = False + test_no_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_no_prefix_path.parts[-1], 'testdirectory') + class MediaTestCase(TestCase): @@ -1661,7 +1680,7 @@ class FormatMatchingTestCase(TestCase): self.media.get_best_audio_format() def test_is_regex_match(self): - + self.media.metadata = all_test_metadata['boring'] self.media.save() expected_matches = { @@ -1689,7 +1708,9 @@ class FormatMatchingTestCase(TestCase): msg=f'Media title "{self.media.title}" checked against regex "{self.source.filter_text}" failed ' f'expected {expected_match_result}') + class TasksTestCase(TestCase): + def setUp(self): # Disable general logging for test case logging.disable(logging.CRITICAL) @@ -1716,31 +1737,3 @@ class TasksTestCase(TestCase): self.assertEqual(src1.media_source.all().count(), 3) self.assertEqual(src2.media_source.all().count(), 2) self.assertEqual(Media.objects.filter(pk=m22.pk).exists(), False) - -class TypeDirectoryPathTestCase(TestCase): - def setUp(self): - self.source = Source( - directory="test_directory", - source_resolution=Source.SOURCE_RESOLUTION_AUDIO, - ) - - def test_directory_prefix_default(self): - """ - Test that default directory prefix exist. - """ - os.environ['TUBESYNC_DIRECTORY_PREFIX'] = '' - self.assertEqual(self.source.type_directory_path, Path(settings.DOWNLOAD_AUDIO_DIR) / 'test_directory') - - def test_directory_prefix_true(self): - """ - Test that when TUBESYNC_DIRECTORY_PREFIX is set to true the directory prefix exist. - """ - os.environ['TUBESYNC_DIRECTORY_PREFIX'] = 'true' - self.assertEqual(self.source.type_directory_path, Path(settings.DOWNLOAD_AUDIO_DIR) / 'test_directory') - - def test_directory_prefix_false(self): - """ - Test that when TUBESYNC_DIRECTORY_PREFIX is set to false the directory prefix does not exist. - """ - os.environ['TUBESYNC_DIRECTORY_PREFIX'] = 'false' - self.assertEqual(self.source.type_directory_path, Path('.') / 'test_directory') \ No newline at end of file diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 18b028b3..96a2e600 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -81,3 +81,6 @@ if BASICAUTH_USERNAME and BASICAUTH_PASSWORD: else: BASICAUTH_DISABLE = True BASICAUTH_USERS = {} + + +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 7bb4b2a2..d08f2cdb 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -158,6 +158,14 @@ VIDEO_HEIGHT_CUTOFF = int(os.getenv("VIDEO_HEIGHT_CUTOFF", "240")) # Smallest r VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' + +# If True source directories are prefixed with their type (either 'video' or 'audio') +# e.g. /downloads/video/SomeSourceName +# If False, sources are placed directly in /downloads +# e.g. /downloads/SomeSourceName +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True + + YOUTUBE_DL_CACHEDIR = None YOUTUBE_DL_TEMPDIR = None YOUTUBE_DEFAULTS = { From a4794ec3b16de72bd9df8b8d061a56beaf87a320 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 11 Dec 2024 06:59:32 -0500 Subject: [PATCH 29/34] Enable filter_source_cutoff with new logic --- tubesync/sync/filtering.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index fb0a6b16..5bb32be3 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -23,8 +23,8 @@ def filter_media(instance: Media): skip = True # Check if older than source_cutoff - #if filter_source_cutoff(instance): - # skip = True + if filter_source_cutoff(instance): + skip = True # Check if we have filter_text and filter text matches if filter_filter_text(instance): @@ -127,20 +127,15 @@ def filter_max_cap(instance: Media): return False -# If the source has a cut-off, check the upload date is within the allowed delta -# TODO: days to keep should be compared to downloaded date, not published +# If the source has a cut-off, check the download date is within the allowed delta def filter_source_cutoff(instance: Media): - if instance.source.delete_old_media and instance.source.days_to_keep > 0: - if not isinstance(instance.published, datetime): - # Media has no known published date or incomplete metadata - log.info( - f"Media: {instance.source} / {instance} has no published date, skipping" - ) - return True + if instance.source.delete_old_media and instance.source.days_to_keep_date: + if not instance.downloaded or not isinstance(instance.download_date, datetime): + return False - delta = timezone.now() - timedelta(days=instance.source.days_to_keep) - if instance.published < delta: - # Media was published after the cutoff date, skip it + days_to_keep_age = instance.source.days_to_keep_date + if instance.download_date < days_to_keep_age: + # Media has expired, skip it log.info( f"Media: {instance.source} / {instance} is older than " f"{instance.source.days_to_keep} days, skipping" From 2c12bf57971fca79df41d19fb6a770580f8ea72e Mon Sep 17 00:00:00 2001 From: meeb Date: Wed, 11 Dec 2024 23:04:14 +1100 Subject: [PATCH 30/34] tidy up settings and readme --- README.md | 39 ++++++++----------- tubesync/tubesync/local_settings.py.container | 3 ++ tubesync/tubesync/settings.py | 4 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9a565c29..a01f9830 100644 --- a/README.md +++ b/README.md @@ -361,30 +361,23 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| --------------------------- | ------------------------------------------------------------ |--------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | -| VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_DIRECTORY_PREFIX | Controls how downloaded files are organized. | true | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | -# TubeSync Directory Mode - -Controls how downloaded files are organized. - -Values: -- true: Audio files go to `audio`, video files to `video`. -- false: All files are placed in the root of DOWNLOAD_DIR. # Manual, non-containerised, installation diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 96a2e600..7b79c5ce 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -84,3 +84,6 @@ else: SOURCE_DOWNLOAD_DIRECTORY_PREFIX = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() + + +VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index d08f2cdb..57ea2bea 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -154,8 +154,8 @@ MEDIA_THUMBNAIL_WIDTH = 430 # Width in pixels to resize thumbnai MEDIA_THUMBNAIL_HEIGHT = 240 # Height in pixels to resize thumbnails to -VIDEO_HEIGHT_CUTOFF = int(os.getenv("VIDEO_HEIGHT_CUTOFF", "240")) # Smallest resolution in pixels permitted to download -VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' +VIDEO_HEIGHT_CUTOFF = 240 # Smallest resolution in pixels permitted to download +VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' From 6c12244a2521ad954b078871a107da1a3cdac66d Mon Sep 17 00:00:00 2001 From: meeb Date: Wed, 11 Dec 2024 23:17:03 +1100 Subject: [PATCH 31/34] whoops --- tubesync/tubesync/local_settings.py.container | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 7b79c5ce..a0426a4c 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -83,7 +83,8 @@ else: BASICAUTH_USERS = {} -SOURCE_DOWNLOAD_DIRECTORY_PREFIX = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() +SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) From 475a26d9dd9cbb0b95f03010137c76ed247f7353 Mon Sep 17 00:00:00 2001 From: meeb Date: Wed, 11 Dec 2024 23:44:28 +1100 Subject: [PATCH 32/34] migration caused by #574 --- .../migrations/0026_alter_source_sub_langs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tubesync/sync/migrations/0026_alter_source_sub_langs.py diff --git a/tubesync/sync/migrations/0026_alter_source_sub_langs.py b/tubesync/sync/migrations/0026_alter_source_sub_langs.py new file mode 100644 index 00000000..937c3a14 --- /dev/null +++ b/tubesync/sync/migrations/0026_alter_source_sub_langs.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-12-11 12:43 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0025_add_video_type_support'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), + ), + ] From de4a15efed372da7ddac6ce6e3b99f394da63d2a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 11 Dec 2024 07:48:39 -0500 Subject: [PATCH 33/34] Remove audio / video directory modification --- tubesync/tubesync/settings.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 57ea2bea..f4865f6a 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -112,10 +112,6 @@ DOWNLOAD_VIDEO_DIR = 'video' DOWNLOAD_AUDIO_DIR = 'audio' SASS_PROCESSOR_ROOT = STATIC_ROOT -directory_prefix = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'true') -if directory_prefix == 'false': - DOWNLOAD_VIDEO_DIR = '.' - DOWNLOAD_AUDIO_DIR = '.' ROBOTS = ''' User-agent: * From d85fb92b2ea2b0fde53a421f0e6c12079aa0baeb Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 17 Dec 2024 15:29:34 -0500 Subject: [PATCH 34/34] Remove unused timedelta import --- tubesync/sync/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index 5bb32be3..45710fe9 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -4,7 +4,7 @@ from common.logger import log from .models import Media -from datetime import datetime, timedelta +from datetime import datetime from django.utils import timezone from .overrides.custom_filter import filter_custom