hoelee/tubesync
1
0
Fork 0
mirror of https://github.com/meeb/tubesync.git synced 2025-06-23 05:26:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'handle_member_only_videos' of github.com:RichardHyde/tubesync into handle_member_only_videos

Browse Source
This commit is contained in:
Richard Hyde 2024-12-21 14:20:23 +00:00
commit 384d8a530d
11 changed files with 155 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -134,3 +134,6 @@ dmypy.json
Pipfile.lock
.vscode/launch.json
# Ignore Jetbrains IDE files
.idea/

16
Dockerfile
View File

@ -3,15 +3,15 @@ FROM debian:bookworm-slim
ARG TARGETARCH
ARG TARGETPLATFORM
ARG S6_VERSION="3.2.0.0"
ARG SHA256_S6_AMD64="ad982a801bd72757c7b1b53539a146cf715e640b4d8f0a6a671a3d1b560fe1e2"
ARG SHA256_S6_ARM64="868973e98210257bba725ff5b17aa092008c9a8e5174499e38ba611a8fc7e473"
ARG SHA256_S6_NOARCH="4b0c0907e6762814c31850e0e6c6762c385571d4656eb8725852b0b1586713b6"
ARG S6_VERSION="3.2.0.2"
ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478"
ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785"
ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
ARG FFMPEG_DATE="autobuild-2024-11-22-14-18"
ARG FFMPEG_VERSION="117857-g2d077f9acd"
ARG SHA256_FFMPEG_AMD64="427ff38cf1e28521aac4fa7931444f6f2ff6f097f2d4a315c6f92ef1d7f90db8"
ARG SHA256_FFMPEG_ARM64="f7ed3a50b651447477aa2637bf8da6010d8f27f8804a5d0e77b00a1d3725b27a"
ARG FFMPEG_DATE="autobuild-2024-12-09-14-16"
ARG FFMPEG_VERSION="118034-gd21134313f"
ARG SHA256_FFMPEG_AMD64="cd50122fb0939e913585282347a8f95074c2d5477ceb059cd90aca551f14e9ea"
ARG SHA256_FFMPEG_ARM64="33b4edebf9c23701473ba8db696b26072bb9b9c05fc4a156e115f94e44d361e0"
ENV S6_VERSION="${S6_VERSION}" \
FFMPEG_DATE="${FFMPEG_DATE}" \

31
README.md
View File

@ -361,21 +361,22 @@ There are a number of other environment variables you can set. These are, mostly
**NOT** required to be set in the default container installation, they are really only
useful if you are manually installing TubeSync in some other environment. These are:
| Name | What | Example |
| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l |
| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ |
| TUBESYNC_DEBUG | Enable debugging | True |
| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 |
| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com |
| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True |
| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 |
| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 |
| LISTEN_PORT | Port number for gunicorn to listen on | 8080 |
| HTTP_USER | Sets the username for HTTP basic authentication | some-username |
| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password |
| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database |
| VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 |
| Name | What | Example |
| ---------------------------- | ------------------------------------------------------------- |--------------------------------------|
| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l |
| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ |
| TUBESYNC_DEBUG | Enable debugging | True |
| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 |
| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com |
| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True |
| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 |
| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True |
| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 |
| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 |
| LISTEN_PORT | Port number for gunicorn to listen on | 8080 |
| HTTP_USER | Sets the username for HTTP basic authentication | some-username |
| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password |
| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database |
# Manual, non-containerised, installation

20
tubesync/sync/filtering.py
View File

@ -4,7 +4,7 @@
from common.logger import log
from .models import Media
from datetime import datetime, timedelta
from datetime import datetime
from django.utils import timezone
from .overrides.custom_filter import filter_custom
@ -127,19 +127,15 @@ def filter_max_cap(instance: Media):
return False
# If the source has a cut-off, check the upload date is within the allowed delta
# If the source has a cut-off, check the download date is within the allowed delta
def filter_source_cutoff(instance: Media):
if instance.source.delete_old_media and instance.source.days_to_keep > 0:
if not isinstance(instance.published, datetime):
# Media has no known published date or incomplete metadata
log.info(
f"Media: {instance.source} / {instance} has no published date, skipping"
)
return True
if instance.source.delete_old_media and instance.source.days_to_keep_date:
if not instance.downloaded or not isinstance(instance.download_date, datetime):
return False
delta = timezone.now() - timedelta(days=instance.source.days_to_keep)
if instance.published < delta:
# Media was published after the cutoff date, skip it
days_to_keep_age = instance.source.days_to_keep_date
if instance.download_date < days_to_keep_age:
# Media has expired, skip it
log.info(
f"Media: {instance.source} / {instance} is older than "
f"{instance.source.days_to_keep} days, skipping"

26
tubesync/sync/matching.py
View File

@ -5,6 +5,7 @@
'''
from .utils import multi_key_sort
from django.conf import settings
@ -21,7 +22,7 @@ def get_best_combined_format(media):
'''
for fmt in media.iter_formats():
# Check height matches
if media.source.source_resolution.strip().upper() != fmt['format']:
if media.source.source_resolution_height != fmt['height']:
continue
# Check the video codec matches
if media.source.source_vcodec != fmt['vcodec']:
@ -47,7 +48,7 @@ def get_best_audio_format(media):
Finds the best match for the source required audio format. If the source
has a 'fallback' of fail this can return no match.
'''
# Order all audio-only formats by bitrate
# Reverse order all audio-only formats
audio_formats = []
for fmt in media.iter_formats():
# If the format has a video stream, skip it
@ -56,18 +57,18 @@ def get_best_audio_format(media):
if not fmt['acodec']:
continue
audio_formats.append(fmt)
audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
if not audio_formats:
# Media has no audio formats at all
return False, False
# Find the highest bitrate audio format with a matching codec
audio_formats = list(reversed(audio_formats))
# Find the first audio format with a matching codec
for fmt in audio_formats:
if media.source.source_acodec == fmt['acodec']:
# Matched!
return True, fmt['id']
# No codecs matched
if media.source.can_fallback:
# Can fallback, find the next highest bitrate non-matching codec
# Can fallback, find the next non-matching codec
return False, audio_formats[0]['id']
else:
# Can't fallback
@ -86,6 +87,7 @@ def get_best_video_format(media):
return False, False
# Filter video-only formats by resolution that matches the source
video_formats = []
sort_keys = [('height', False), ('vcodec', True), ('vbr', False)] # key, reverse
for fmt in media.iter_formats():
# If the format has an audio stream, skip it
if fmt['acodec'] is not None:
@ -94,6 +96,8 @@ def get_best_video_format(media):
continue
if media.source.source_resolution.strip().upper() == fmt['format']:
video_formats.append(fmt)
elif media.source.source_resolution_height == fmt['height']:
video_formats.append(fmt)
# Check we matched some streams
if not video_formats:
# No streams match the requested resolution, see if we can fallback
@ -109,13 +113,17 @@ def get_best_video_format(media):
else:
# Can't fallback
return False, False
video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height'])))
source_resolution = media.source.source_resolution.strip().upper()
source_vcodec = media.source.source_vcodec
if not video_formats:
# Still no matches
return False, False
video_formats = multi_key_sort(video_formats, sort_keys, True)
source_resolution = media.source.source_resolution.strip().upper()
source_vcodec = media.source.source_vcodec
exact_match, best_match = None, None
for fmt in video_formats:
# format_note was blank, match height instead
if '' == fmt['format'] and fmt['height'] == media.source.source_resolution_height:
fmt['format'] = source_resolution
# Of our filtered video formats, check for resolution + codec + hdr + fps match
if media.source.prefer_60fps and media.source.prefer_hdr:
for fmt in video_formats:
@ -331,7 +339,7 @@ def get_best_video_format(media):
for fmt in video_formats:
# Check for a codec, hdr and fps match but drop the resolution
if (source_vcodec == fmt['vcodec'] and
not fmt['is_hdr'] and fmt['is_60fps']):
not fmt['is_hdr'] and not fmt['is_60fps']):
# Close match
exact_match, best_match = False, fmt
break

19
tubesync/sync/migrations/0026_alter_source_sub_langs.py Normal file
View File

@ -0,0 +1,19 @@
# Generated by Django 3.2.25 on 2024-12-11 12:43
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0025_add_video_type_support'),
]
operations = [
migrations.AlterField(
model_name='source',
name='sub_langs',
field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'),
),
]

24
tubesync/sync/models.py
View File

@ -422,7 +422,7 @@ class Source(models.Model):
help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
validators=[
RegexValidator(
regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+",
message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
)
]
@ -460,6 +460,14 @@ class Source(models.Model):
else:
return False
@property
def days_to_keep_date(self):
delta = self.days_to_keep
if delta > 0:
return timezone.now() - timedelta(days=delta)
else:
return False
@property
def extension(self):
'''
@ -514,10 +522,13 @@ class Source(models.Model):
@property
def type_directory_path(self):
if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX:
if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
else:
return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
else:
return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
return Path(self.directory)
def make_directory(self):
return os.makedirs(self.directory_path, exist_ok=True)
@ -1291,10 +1302,7 @@ class Media(models.Model):
@property
def directory_path(self):
# Otherwise, create a suitable filename from the source media_format
media_format = str(self.source.media_format)
media_details = self.format_dict
dirname = self.source.directory_path / media_format.format(**media_details)
dirname = self.source.directory_path / self.filename
return os.path.dirname(str(dirname))
@property

25
tubesync/sync/tests.py
View File

@ -6,7 +6,9 @@
import logging
import os
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
from xml.etree import ElementTree
from django.conf import settings
@ -626,6 +628,25 @@ class FilepathTestCase(TestCase):
('no-fancy-stuff-title_test_720p-720x1280-opus'
'-vp9-30fps-hdr.mkv'))
def test_directory_prefix(self):
# Confirm the setting exists and is valid
self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX'))
self.assertTrue(isinstance(settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX, bool))
# Test the default behavior for "True", forced "audio" or "video" parent directories for sources
settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True
self.source.source_resolution = Source.SOURCE_RESOLUTION_AUDIO
test_audio_prefix_path = Path(self.source.directory_path)
self.assertEqual(test_audio_prefix_path.parts[-2], 'audio')
self.assertEqual(test_audio_prefix_path.parts[-1], 'testdirectory')
self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P
test_video_prefix_path = Path(self.source.directory_path)
self.assertEqual(test_video_prefix_path.parts[-2], 'video')
self.assertEqual(test_video_prefix_path.parts[-1], 'testdirectory')
# Test the default behavior for "False", no parent directories for sources
settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = False
test_no_prefix_path = Path(self.source.directory_path)
self.assertEqual(test_no_prefix_path.parts[-1], 'testdirectory')
class MediaTestCase(TestCase):
@ -1659,7 +1680,7 @@ class FormatMatchingTestCase(TestCase):
self.media.get_best_audio_format()
def test_is_regex_match(self):
self.media.metadata = all_test_metadata['boring']
self.media.save()
expected_matches = {
@ -1687,7 +1708,9 @@ class FormatMatchingTestCase(TestCase):
msg=f'Media title "{self.media.title}" checked against regex "{self.source.filter_text}" failed '
f'expected {expected_match_result}')
class TasksTestCase(TestCase):
def setUp(self):
# Disable general logging for test case
logging.disable(logging.CRITICAL)

41
tubesync/sync/utils.py
View File

@ -1,6 +1,7 @@
import os
import re
import math
from operator import itemgetter
from pathlib import Path
import requests
from PIL import Image
@ -134,6 +135,30 @@ def seconds_to_timestr(seconds):
return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
def multi_key_sort(sort_dict, specs, use_reversed=False):
result = list(sort_dict)
for key, reverse in reversed(specs):
result = sorted(result, key=itemgetter(key), reverse=reverse)
if use_reversed:
return list(reversed(result))
return result
def normalize_codec(codec_str):
result = str(codec_str).upper()
parts = result.split('.')
if len(parts) > 0:
result = parts[0].strip()
else:
return None
if 'NONE' == result:
return None
if str(0) in result:
prefix = result.rstrip('0123456789')
result = prefix + str(int(result[len(prefix):]))
return result
def parse_media_format(format_dict):
'''
This parser primarily adapts the format dict returned by youtube-dl into a
@ -141,21 +166,9 @@ def parse_media_format(format_dict):
any internals, update it here.
'''
vcodec_full = format_dict.get('vcodec', '')
vcodec_parts = vcodec_full.split('.')
if len(vcodec_parts) > 0:
vcodec = vcodec_parts[0].strip().upper()
else:
vcodec = None
if vcodec == 'NONE':
vcodec = None
vcodec = normalize_codec(vcodec_full)
acodec_full = format_dict.get('acodec', '')
acodec_parts = acodec_full.split('.')
if len(acodec_parts) > 0:
acodec = acodec_parts[0].strip().upper()
else:
acodec = None
if acodec == 'NONE':
acodec = None
acodec = normalize_codec(acodec_full)
try:
fps = int(format_dict.get('fps', 0))
except (ValueError, TypeError):

7
tubesync/tubesync/local_settings.py.container
View File

@ -81,3 +81,10 @@ if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
else:
BASICAUTH_DISABLE = True
BASICAUTH_USERS = {}
SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower()
SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False
VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))

12
tubesync/tubesync/settings.py
View File

@ -150,8 +150,16 @@ MEDIA_THUMBNAIL_WIDTH = 430 # Width in pixels to resize thumbnai
MEDIA_THUMBNAIL_HEIGHT = 240 # Height in pixels to resize thumbnails to
VIDEO_HEIGHT_CUTOFF = int(os.getenv("VIDEO_HEIGHT_CUTOFF", "240")) # Smallest resolution in pixels permitted to download
VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD'
VIDEO_HEIGHT_CUTOFF = 240 # Smallest resolution in pixels permitted to download
VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD'
# If True source directories are prefixed with their type (either 'video' or 'audio')
# e.g. /downloads/video/SomeSourceName
# If False, sources are placed directly in /downloads
# e.g. /downloads/SomeSourceName
SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True
YOUTUBE_DL_CACHEDIR = None