Merge branch 'handle_member_only_videos' of github.com:RichardHyde/tubesync into handle_member_only_videos

.gitignore

@@ -134,3 +134,6 @@ dmypy.json
 
 Pipfile.lock
 .vscode/launch.json
+
+# Ignore Jetbrains IDE files
+.idea/

Dockerfile

@@ -3,15 +3,15 @@ FROM debian:bookworm-slim
 ARG TARGETARCH
 ARG TARGETPLATFORM
 
-ARG S6_VERSION="3.2.0.0"
+ARG S6_VERSION="3.2.0.2"
-ARG SHA256_S6_AMD64="ad982a801bd72757c7b1b53539a146cf715e640b4d8f0a6a671a3d1b560fe1e2"
+ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478"
-ARG SHA256_S6_ARM64="868973e98210257bba725ff5b17aa092008c9a8e5174499e38ba611a8fc7e473"
+ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785"
-ARG SHA256_S6_NOARCH="4b0c0907e6762814c31850e0e6c6762c385571d4656eb8725852b0b1586713b6"
+ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
 
-ARG FFMPEG_DATE="autobuild-2024-11-22-14-18"
+ARG FFMPEG_DATE="autobuild-2024-12-09-14-16"
-ARG FFMPEG_VERSION="117857-g2d077f9acd"
+ARG FFMPEG_VERSION="118034-gd21134313f"
-ARG SHA256_FFMPEG_AMD64="427ff38cf1e28521aac4fa7931444f6f2ff6f097f2d4a315c6f92ef1d7f90db8"
+ARG SHA256_FFMPEG_AMD64="cd50122fb0939e913585282347a8f95074c2d5477ceb059cd90aca551f14e9ea"
-ARG SHA256_FFMPEG_ARM64="f7ed3a50b651447477aa2637bf8da6010d8f27f8804a5d0e77b00a1d3725b27a"
+ARG SHA256_FFMPEG_ARM64="33b4edebf9c23701473ba8db696b26072bb9b9c05fc4a156e115f94e44d361e0"
 
 ENV S6_VERSION="${S6_VERSION}" \
   FFMPEG_DATE="${FFMPEG_DATE}" \

README.md

@@ -362,20 +362,21 @@ There are a number of other environment variables you can set. These are, mostly
 useful if you are manually installing TubeSync in some other environment. These are:
 
 | Name                         | What                                                          | Example                              |
-| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
+| ---------------------------- | ------------------------------------------------------------- |--------------------------------------|
 | DJANGO_SECRET_KEY            | Django's SECRET_KEY                                           | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
 | DJANGO_URL_PREFIX            | Run TubeSync in a sub-URL on the web server                   | /somepath/                           |
 | TUBESYNC_DEBUG               | Enable debugging                                              | True                                 |
 | TUBESYNC_WORKERS             | Number of background workers, default is 2, max allowed is 8  | 2                                    |
 | TUBESYNC_HOSTS               | Django's ALLOWED_HOSTS, defaults to `*`                       | tubesync.example.com,otherhost.com   |
 | TUBESYNC_RESET_DOWNLOAD_DIR  | Toggle resetting `/downloads` permissions, defaults to True   | True                                 |
+| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download         | 240                                  |
+| TUBESYNC_DIRECTORY_PREFIX    | Enable `video` and `audio` directory prefixes in `/downloads` | True                                 |
 | GUNICORN_WORKERS             | Number of gunicorn workers to spawn                           | 3                                    |
 | LISTEN_HOST                  | IP address for gunicorn to listen on                          | 127.0.0.1                            |
 | LISTEN_PORT                  | Port number for gunicorn to listen on                         | 8080                                 |
 | HTTP_USER                    | Sets the username for HTTP basic authentication               | some-username                        |
 | HTTP_PASS                    | Sets the password for HTTP basic authentication               | some-secure-password                 |
 | DATABASE_CONNECTION          | Optional external database connection details                 | mysql://user:pass@host:port/database |
-| VIDEO_HEIGHT_CUTOFF         | Smallest video height in pixels permitted to download        | 240                                  |
 
 
 # Manual, non-containerised, installation

tubesync/sync/filtering.py

@@ -4,7 +4,7 @@
 
 from common.logger import log
 from .models import Media
-from datetime import datetime, timedelta
+from datetime import datetime
 from django.utils import timezone
 from .overrides.custom_filter import filter_custom
 
@@ -127,19 +127,15 @@ def filter_max_cap(instance: Media):
     return False
 
 
-# If the source has a cut-off, check the upload date is within the allowed delta
+# If the source has a cut-off, check the download date is within the allowed delta
 def filter_source_cutoff(instance: Media):
-    if instance.source.delete_old_media and instance.source.days_to_keep > 0:
+    if instance.source.delete_old_media and instance.source.days_to_keep_date:
-        if not isinstance(instance.published, datetime):
+        if not instance.downloaded or not isinstance(instance.download_date, datetime):
-            # Media has no known published date or incomplete metadata
+            return False
-            log.info(
-                f"Media: {instance.source} / {instance} has no published date, skipping"
-            )
-            return True
 
-        delta = timezone.now() - timedelta(days=instance.source.days_to_keep)
+        days_to_keep_age = instance.source.days_to_keep_date
-        if instance.published < delta:
+        if instance.download_date < days_to_keep_age:
-            # Media was published after the cutoff date, skip it
+            # Media has expired, skip it
             log.info(
                 f"Media: {instance.source} / {instance} is older than "
                 f"{instance.source.days_to_keep} days, skipping"

tubesync/sync/matching.py

@@ -5,6 +5,7 @@
 '''
 
 
+from .utils import multi_key_sort
 from django.conf import settings
 
 
@@ -21,7 +22,7 @@ def get_best_combined_format(media):
     '''
     for fmt in media.iter_formats():
         # Check height matches
-        if media.source.source_resolution.strip().upper() != fmt['format']:
+        if media.source.source_resolution_height != fmt['height']:
             continue
         # Check the video codec matches
         if media.source.source_vcodec != fmt['vcodec']:
@@ -47,7 +48,7 @@ def get_best_audio_format(media):
         Finds the best match for the source required audio format. If the source
         has a 'fallback' of fail this can return no match.
     '''
-    # Order all audio-only formats by bitrate
+    # Reverse order all audio-only formats
     audio_formats = []
     for fmt in media.iter_formats():
         # If the format has a video stream, skip it
@@ -56,18 +57,18 @@ def get_best_audio_format(media):
         if not fmt['acodec']:
             continue
         audio_formats.append(fmt)
-    audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
     if not audio_formats:
         # Media has no audio formats at all
         return False, False
-    # Find the highest bitrate audio format with a matching codec
+    audio_formats = list(reversed(audio_formats))
+    # Find the first audio format with a matching codec
     for fmt in audio_formats:
         if media.source.source_acodec == fmt['acodec']:
             # Matched!
             return True, fmt['id']
     # No codecs matched
     if media.source.can_fallback:
-        # Can fallback, find the next highest bitrate non-matching codec
+        # Can fallback, find the next non-matching codec
         return False, audio_formats[0]['id']
     else:
         # Can't fallback
@@ -86,6 +87,7 @@ def get_best_video_format(media):
         return False, False
     # Filter video-only formats by resolution that matches the source
     video_formats = []
+    sort_keys = [('height', False), ('vcodec', True), ('vbr', False)] # key, reverse
     for fmt in media.iter_formats():
         # If the format has an audio stream, skip it
         if fmt['acodec'] is not None:
@@ -94,6 +96,8 @@ def get_best_video_format(media):
             continue
         if media.source.source_resolution.strip().upper() == fmt['format']:
             video_formats.append(fmt)
+        elif media.source.source_resolution_height == fmt['height']:
+            video_formats.append(fmt)
     # Check we matched some streams
     if not video_formats:
         # No streams match the requested resolution, see if we can fallback
@@ -109,13 +113,17 @@ def get_best_video_format(media):
         else:
             # Can't fallback
             return False, False
-    video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height'])))
-    source_resolution = media.source.source_resolution.strip().upper()
-    source_vcodec = media.source.source_vcodec
     if not video_formats:
         # Still no matches
         return False, False
+    video_formats = multi_key_sort(video_formats, sort_keys, True)
+    source_resolution = media.source.source_resolution.strip().upper()
+    source_vcodec = media.source.source_vcodec
     exact_match, best_match = None, None
+    for fmt in video_formats:
+        # format_note was blank, match height instead
+        if '' == fmt['format'] and fmt['height'] == media.source.source_resolution_height:
+            fmt['format'] = source_resolution
     # Of our filtered video formats, check for resolution + codec + hdr + fps match
     if media.source.prefer_60fps and media.source.prefer_hdr:
         for fmt in video_formats:
@@ -331,7 +339,7 @@ def get_best_video_format(media):
                 for fmt in video_formats:
                     # Check for a codec, hdr and fps match but drop the resolution
                     if (source_vcodec == fmt['vcodec'] and 
-                        not fmt['is_hdr'] and fmt['is_60fps']):
+                        not fmt['is_hdr'] and not fmt['is_60fps']):
                         # Close match
                         exact_match, best_match = False, fmt
                         break

tubesync/sync/migrations/0026_alter_source_sub_langs.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-12-11 12:43
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0025_add_video_type_support'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='sub_langs',
+            field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'),
+        ),
+    ]

tubesync/sync/models.py

@@ -422,7 +422,7 @@ class Source(models.Model):
         help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
         validators=[
             RegexValidator(
-                regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
+                regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+",
                 message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
             )
         ]
@@ -460,6 +460,14 @@ class Source(models.Model):
         else:
             return False
 
+    @property
+    def days_to_keep_date(self):
+        delta = self.days_to_keep
+        if delta > 0:
+            return timezone.now() - timedelta(days=delta)
+        else:
+            return False
+
     @property
     def extension(self):
         '''
@@ -514,10 +522,13 @@ class Source(models.Model):
 
     @property
     def type_directory_path(self):
+        if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX:
             if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
                 return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
             else:
                 return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
+        else:
+            return Path(self.directory)
 
     def make_directory(self):
         return os.makedirs(self.directory_path, exist_ok=True)
@@ -1291,10 +1302,7 @@ class Media(models.Model):
 
     @property
     def directory_path(self):
-        # Otherwise, create a suitable filename from the source media_format
+        dirname = self.source.directory_path / self.filename
-        media_format = str(self.source.media_format)
-        media_details = self.format_dict
-        dirname = self.source.directory_path / media_format.format(**media_details)
         return os.path.dirname(str(dirname))
 
     @property

tubesync/sync/tests.py

@@ -6,7 +6,9 @@
 
 
 import logging
+import os
 from datetime import datetime, timedelta
+from pathlib import Path
 from urllib.parse import urlsplit
 from xml.etree import ElementTree
 from django.conf import settings
@@ -626,6 +628,25 @@ class FilepathTestCase(TestCase):
                          ('no-fancy-stuff-title_test_720p-720x1280-opus'
                           '-vp9-30fps-hdr.mkv'))
 
+    def test_directory_prefix(self):
+        # Confirm the setting exists and is valid
+        self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX'))
+        self.assertTrue(isinstance(settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX, bool))
+        # Test the default behavior for "True", forced "audio" or "video" parent directories for sources
+        settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True
+        self.source.source_resolution = Source.SOURCE_RESOLUTION_AUDIO
+        test_audio_prefix_path = Path(self.source.directory_path)
+        self.assertEqual(test_audio_prefix_path.parts[-2], 'audio')
+        self.assertEqual(test_audio_prefix_path.parts[-1], 'testdirectory')
+        self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P
+        test_video_prefix_path = Path(self.source.directory_path)
+        self.assertEqual(test_video_prefix_path.parts[-2], 'video')
+        self.assertEqual(test_video_prefix_path.parts[-1], 'testdirectory')
+        # Test the default behavior for "False", no parent directories for sources
+        settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = False
+        test_no_prefix_path = Path(self.source.directory_path)
+        self.assertEqual(test_no_prefix_path.parts[-1], 'testdirectory')
+
 
 class MediaTestCase(TestCase):
 
@@ -1687,7 +1708,9 @@ class FormatMatchingTestCase(TestCase):
                 msg=f'Media title "{self.media.title}" checked against regex "{self.source.filter_text}" failed '
                     f'expected {expected_match_result}')
 
+
 class TasksTestCase(TestCase):
+
     def setUp(self):
         # Disable general logging for test case
         logging.disable(logging.CRITICAL)

tubesync/sync/utils.py

@@ -1,6 +1,7 @@
 import os
 import re
 import math
+from operator import itemgetter
 from pathlib import Path
 import requests
 from PIL import Image
@@ -134,6 +135,30 @@ def seconds_to_timestr(seconds):
    return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
 
 
+def multi_key_sort(sort_dict, specs, use_reversed=False):
+    result = list(sort_dict)
+    for key, reverse in reversed(specs):
+        result = sorted(result, key=itemgetter(key), reverse=reverse)
+    if use_reversed:
+        return list(reversed(result))
+    return result
+
+
+def normalize_codec(codec_str):
+    result = str(codec_str).upper()
+    parts = result.split('.')
+    if len(parts) > 0:
+        result = parts[0].strip()
+    else:
+        return None
+    if 'NONE' == result:
+        return None
+    if str(0) in result:
+        prefix = result.rstrip('0123456789')
+        result = prefix + str(int(result[len(prefix):]))
+    return result
+
+
 def parse_media_format(format_dict):
     '''
         This parser primarily adapts the format dict returned by youtube-dl into a
@@ -141,21 +166,9 @@ def parse_media_format(format_dict):
         any internals, update it here.
     '''
     vcodec_full = format_dict.get('vcodec', '')
-    vcodec_parts = vcodec_full.split('.')
+    vcodec = normalize_codec(vcodec_full)
-    if len(vcodec_parts) > 0:
-        vcodec = vcodec_parts[0].strip().upper()
-    else:
-        vcodec = None
-    if vcodec == 'NONE':
-        vcodec = None
     acodec_full = format_dict.get('acodec', '')
-    acodec_parts = acodec_full.split('.')
+    acodec = normalize_codec(acodec_full) 
-    if len(acodec_parts) > 0:
-        acodec = acodec_parts[0].strip().upper()
-    else:
-        acodec = None
-    if acodec == 'NONE':
-        acodec = None
     try:
         fps = int(format_dict.get('fps', 0))
     except (ValueError, TypeError):

tubesync/tubesync/local_settings.py.container

@@ -81,3 +81,10 @@ if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
 else:
     BASICAUTH_DISABLE = True
     BASICAUTH_USERS = {}
+
+
+SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower()
+SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False
+
+
+VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))

tubesync/tubesync/settings.py

@@ -150,10 +150,18 @@ MEDIA_THUMBNAIL_WIDTH = 430                 # Width in pixels to resize thumbnai
 MEDIA_THUMBNAIL_HEIGHT = 240                # Height in pixels to resize thumbnails to
 
 
-VIDEO_HEIGHT_CUTOFF = int(os.getenv("VIDEO_HEIGHT_CUTOFF", "240"))  # Smallest resolution in pixels permitted to download
+VIDEO_HEIGHT_CUTOFF = 240                   # Smallest resolution in pixels permitted to download
 VIDEO_HEIGHT_IS_HD = 500                    # Height in pixels to count as 'HD'
 
 
+
+# If True source directories are prefixed with their type (either 'video' or 'audio')
+# e.g. /downloads/video/SomeSourceName
+# If False, sources are placed directly in /downloads
+# e.g. /downloads/SomeSourceName
+SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True
+
+
 YOUTUBE_DL_CACHEDIR = None
 YOUTUBE_DL_TEMPDIR = None
 YOUTUBE_DEFAULTS = {