| Available formats |
Available formats
- {% for format in media.formats %}
+ {% for format in media.iter_formats %}
- ID: {{ format.format_id }}
- {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %}
- {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% endif %}
- {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}(matched){% endif %}
+ ID: {{ format.id }}
+ {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.vbr }}k{% endif %}
+ {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} {% if format.abr %}@{{ format.abr }}k / {% endif %}{{ format.asr|intcomma }}Hz{% if format.language_code %} [{{ format.language_code }}]{% endif %}{% if format.abr %} {{ format.format_note }}{% endif %}{% endif %}
+ {% if format.id == combined_format or format.id == audio_format or format.id == video_format %}(matched){% endif %}
{% empty %}
Media has no indexed available formats
diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 8f0de6ef..2704058f 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -18,6 +18,7 @@ from background_task.models import Task
from .models import Source, Media
from .tasks import cleanup_old_media
from .filtering import filter_media
+from .utils import filter_response
class FrontEndTestCase(TestCase):
@@ -1709,6 +1710,84 @@ class FormatMatchingTestCase(TestCase):
f'expected {expected_match_result}')
+class ResponseFilteringTestCase(TestCase):
+
+ def setUp(self):
+ # Disable general logging for test case
+ logging.disable(logging.CRITICAL)
+ # Add a test source
+ self.source = Source.objects.create(
+ source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
+ key='testkey',
+ name='testname',
+ directory='testdirectory',
+ index_schedule=3600,
+ delete_old_media=False,
+ days_to_keep=14,
+ source_resolution=Source.SOURCE_RESOLUTION_1080P,
+ source_vcodec=Source.SOURCE_VCODEC_VP9,
+ source_acodec=Source.SOURCE_ACODEC_OPUS,
+ prefer_60fps=False,
+ prefer_hdr=False,
+ fallback=Source.FALLBACK_FAIL
+ )
+ # Add some media
+ self.media = Media.objects.create(
+ key='mediakey',
+ source=self.source,
+ metadata='{}'
+ )
+
+ def test_metadata_20230629(self):
+ self.media.metadata = all_test_metadata['20230629']
+ self.media.save()
+
+ unfiltered = self.media.loaded_metadata
+ filtered = filter_response(self.media.loaded_metadata)
+ self.assertIn('formats', unfiltered.keys())
+ self.assertIn('formats', filtered.keys())
+ # filtered 'downloader_options'
+ self.assertIn('downloader_options', unfiltered['formats'][10].keys())
+ self.assertNotIn('downloader_options', filtered['formats'][10].keys())
+ # filtered 'http_headers'
+ self.assertIn('http_headers', unfiltered['formats'][0].keys())
+ self.assertNotIn('http_headers', filtered['formats'][0].keys())
+ # did not lose any formats
+ self.assertEqual(48, len(unfiltered['formats']))
+ self.assertEqual(48, len(filtered['formats']))
+ self.assertEqual(len(unfiltered['formats']), len(filtered['formats']))
+ # did not remove everything with url
+ self.assertIn('original_url', unfiltered.keys())
+ self.assertIn('original_url', filtered.keys())
+ self.assertEqual(unfiltered['original_url'], filtered['original_url'])
+ # did reduce the size of the metadata
+ self.assertTrue(len(str(filtered)) < len(str(unfiltered)))
+
+ url_keys = []
+ for format in unfiltered['formats']:
+ for key in format.keys():
+ if 'url' in key:
+ url_keys.append((format['format_id'], key, format[key],))
+ unfiltered_url_keys = url_keys
+ self.assertEqual(63, len(unfiltered_url_keys), msg=str(unfiltered_url_keys))
+
+ url_keys = []
+ for format in filtered['formats']:
+ for key in format.keys():
+ if 'url' in key:
+ url_keys.append((format['format_id'], key, format[key],))
+ filtered_url_keys = url_keys
+ self.assertEqual(3, len(filtered_url_keys), msg=str(filtered_url_keys))
+
+ url_keys = []
+ for lang_code, captions in filtered['automatic_captions'].items():
+ for caption in captions:
+ for key in caption.keys():
+ if 'url' in key:
+ url_keys.append((lang_code, caption['ext'], caption[key],))
+ self.assertEqual(0, len(url_keys), msg=str(url_keys))
+
+
class TasksTestCase(TestCase):
def setUp(self):
diff --git a/tubesync/sync/urls.py b/tubesync/sync/urls.py
index 66558379..116beca5 100644
--- a/tubesync/sync/urls.py
+++ b/tubesync/sync/urls.py
@@ -17,7 +17,7 @@ urlpatterns = [
path('',
DashboardView.as_view(),
name='dashboard'),
-
+
# Source URLs
path('sources',
diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index e1ab6142..2f9e6593 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -1,6 +1,7 @@
import os
import re
import math
+from copy import deepcopy
from operator import itemgetter
from pathlib import Path
from tempfile import NamedTemporaryFile
@@ -128,6 +129,23 @@ def file_is_editable(filepath):
return False
+def directory_and_stem(arg_path):
+ filepath = Path(arg_path)
+ stem = Path(filepath.stem)
+ while stem.suffixes and '' != stem.suffix:
+ stem = Path(stem.stem)
+ stem = str(stem)
+ return (filepath.parent, stem,)
+
+
+def mkdir_p(arg_path, mode=0o777):
+ '''
+ Reminder: mode only affects the last directory
+ '''
+ dirpath = Path(arg_path)
+ return dirpath.mkdir(mode=mode, parents=True, exist_ok=True)
+
+
def write_text_file(filepath, filedata):
if not isinstance(filedata, str):
raise TypeError(f'filedata must be a str, got "{type(filedata)}"')
@@ -137,7 +155,8 @@ def write_text_file(filepath, filedata):
bytes_written = f.write(filedata)
# chmod a+r temp_file
old_mode = new_filepath.stat().st_mode
- new_filepath.chmod(0o444 | old_mode)
+ if 0o444 != (0o444 & old_mode):
+ new_filepath.chmod(0o444 | old_mode)
if not file_is_editable(new_filepath):
new_filepath.unlink()
raise ValueError(f'File cannot be edited or removed: {filepath}')
@@ -184,6 +203,95 @@ def normalize_codec(codec_str):
return result
+def _url_keys(arg_dict, filter_func):
+ result = {}
+ for key in arg_dict.keys():
+ if 'url' in key:
+ result.update(
+ {key: filter_func(key=key, url=arg_dict[key])}
+ )
+ return result
+
+
+def _drop_url_keys(arg_dict, key, filter_func):
+ if key in arg_dict.keys():
+ for val_dict in arg_dict[key]:
+ for url_key, remove in _url_keys(val_dict, filter_func).items():
+ if remove is True:
+ del val_dict[url_key]
+
+
+def filter_response(arg_dict, copy_arg=False):
+ '''
+ Clean up the response so as to not store useless metadata in the database.
+ '''
+ response_dict = arg_dict
+ # raise an exception for an unexpected argument type
+ if not isinstance(response_dict, dict):
+ raise TypeError(f'response_dict must be a dict, got "{type(response_dict)}"')
+
+ if copy_arg:
+ response_dict = deepcopy(arg_dict)
+
+ # optimize the empty case
+ if not response_dict:
+ return response_dict
+
+ # beginning of formats cleanup {{{
+ # drop urls that expire, or restrict IPs
+ def drop_format_url(**kwargs):
+ url = kwargs['url']
+ return (
+ url
+ and '://' in url
+ and (
+ '/ip/' in url
+ or 'ip=' in url
+ or '/expire/' in url
+ or 'expire=' in url
+ )
+ )
+
+ # these format keys are not useful to us
+ drop_keys = frozenset((
+ 'downloader_options',
+ 'fragments',
+ 'http_headers',
+ '__needs_testing',
+ '__working',
+ ))
+ for key in frozenset(('formats', 'requested_formats',)):
+ _drop_url_keys(response_dict, key, drop_format_url)
+ if key in response_dict.keys():
+ for format in response_dict[key]:
+ for drop_key in drop_keys:
+ if drop_key in format.keys():
+ del format[drop_key]
+ # end of formats cleanup }}}
+
+ # beginning of subtitles cleanup {{{
+ # drop urls that expire
+ def drop_subtitles_url(**kwargs):
+ url = kwargs['url']
+ return (
+ url
+ and '://' in url
+ and (
+ '/expire/' in url
+ or '&expire=' in url
+ )
+ )
+
+ for key in frozenset(('subtitles', 'automatic_captions',)):
+ if key in response_dict.keys():
+ key_dict = response_dict[key]
+ for lang_code in key_dict:
+ _drop_url_keys(key_dict, lang_code, drop_subtitles_url)
+ # end of subtitles cleanup }}}
+
+ return response_dict
+
+
def parse_media_format(format_dict):
'''
This parser primarily adapts the format dict returned by youtube-dl into a
@@ -224,10 +332,13 @@ def parse_media_format(format_dict):
format_str = f'{height}P'
else:
format_str = None
+
return {
'id': format_dict.get('format_id', ''),
'format': format_str,
+ 'format_note': format_dict.get('format_note', ''),
'format_verbose': format_dict.get('format', ''),
+ 'language_code': format_dict.get('language', None),
'height': height,
'width': width,
'vcodec': vcodec,
@@ -235,6 +346,7 @@ def parse_media_format(format_dict):
'vbr': format_dict.get('tbr', 0),
'acodec': acodec,
'abr': format_dict.get('abr', 0),
+ 'asr': format_dict.get('asr', 0),
'is_60fps': fps > 50,
'is_hdr': 'HDR' in format_dict.get('format', '').upper(),
'is_hls': is_hls,
diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 52090042..24ff6866 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -14,7 +14,7 @@ from django.views.generic.detail import SingleObjectMixin
from django.core.exceptions import SuspiciousFileOperation
from django.http import HttpResponse
from django.urls import reverse_lazy
-from django.db import IntegrityError
+from django.db import connection, IntegrityError
from django.db.models import Q, Count, Sum, When, Case
from django.forms import Form, ValidationError
from django.utils.text import slugify
@@ -85,6 +85,12 @@ class DashboardView(TemplateView):
data['config_dir'] = str(settings.CONFIG_BASE_DIR)
data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
data['database_connection'] = settings.DATABASE_CONNECTION_STR
+ # Add the database filesize when using db.sqlite3
+ data['database_filesize'] = None
+ db_name = str(connection.get_connection_params()['database'])
+ db_path = pathlib.Path(db_name) if '/' == db_name[0] else None
+ if db_path and 'sqlite' == connection.vendor:
+ data['database_filesize'] = db_path.stat().st_size
return data
@@ -106,7 +112,7 @@ class SourcesView(ListView):
sobj = Source.objects.get(pk=kwargs["pk"])
if sobj is None:
return HttpResponseNotFound()
-
+
verbose_name = _('Index media from source "{}" once')
index_source_task(
str(sobj.pk),
@@ -193,10 +199,15 @@ class ValidateSourceView(FormView):
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r')
}
+ _youtube_domains = frozenset({
+ 'youtube.com',
+ 'm.youtube.com',
+ 'www.youtube.com',
+ })
validation_urls = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
'scheme': 'https',
- 'domains': ('m.youtube.com', 'www.youtube.com'),
+ 'domains': _youtube_domains,
'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
'path_must_not_match': ('/playlist', '/c/playlist'),
'qs_args': [],
@@ -205,7 +216,7 @@ class ValidateSourceView(FormView):
},
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
'scheme': 'https',
- 'domains': ('m.youtube.com', 'www.youtube.com'),
+ 'domains': _youtube_domains,
'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
'path_must_not_match': ('/playlist', '/c/playlist'),
'qs_args': [],
@@ -214,7 +225,7 @@ class ValidateSourceView(FormView):
},
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
'scheme': 'https',
- 'domains': ('m.youtube.com', 'www.youtube.com'),
+ 'domains': _youtube_domains,
'path_regex': '^\/(playlist|watch)$',
'path_must_not_match': (),
'qs_args': ('list',),
@@ -286,11 +297,36 @@ class ValidateSourceView(FormView):
url = reverse_lazy('sync:add-source')
fields_to_populate = self.prepopulate_fields.get(self.source_type)
fields = {}
+ value = self.key
+ use_channel_id = (
+ 'youtube-channel' == self.source_type_str and
+ '@' == self.key[0]
+ )
+ if use_channel_id:
+ old_key = self.key
+ old_source_type = self.source_type
+ old_source_type_str = self.source_type_str
+
+ self.source_type_str = 'youtube-channel-id'
+ self.source_type = self.source_types.get(self.source_type_str, None)
+ index_url = Source.create_index_url(self.source_type, self.key, 'videos')
+ try:
+ self.key = youtube.get_channel_id(
+ index_url.replace('/channel/', '/')
+ )
+ except youtube.YouTubeError as e:
+ # It did not work, revert to previous behavior
+ self.key = old_key
+ self.source_type = old_source_type
+ self.source_type_str = old_source_type_str
+
for field in fields_to_populate:
if field == 'source_type':
fields[field] = self.source_type
- elif field in ('key', 'name', 'directory'):
+ elif field == 'key':
fields[field] = self.key
+ elif field in ('name', 'directory'):
+ fields[field] = value
return append_uri_params(url, fields)
@@ -318,7 +354,7 @@ class EditSourceMixin:
obj = form.save(commit=False)
source_type = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
-
+
if example_media_file == '':
form.add_error(
'media_format',
@@ -488,20 +524,15 @@ class MediaView(ListView):
return super().dispatch(request, *args, **kwargs)
def get_queryset(self):
+ q = Media.objects.all()
+
if self.filter_source:
- if self.show_skipped:
- q = Media.objects.filter(source=self.filter_source)
- elif self.only_skipped:
- q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
- else:
- q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
- else:
- if self.show_skipped:
- q = Media.objects.all()
- elif self.only_skipped:
- q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
- else:
- q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
+ q = q.filter(source=self.filter_source)
+ if self.only_skipped:
+ q = q.filter(Q(can_download=False) | Q(skip=True) | Q(manual_skip=True))
+ elif not self.show_skipped:
+ q = q.filter(Q(can_download=True) & Q(skip=False) & Q(manual_skip=False))
+
return q.order_by('-published', '-created')
def get_context_data(self, *args, **kwargs):
@@ -740,18 +771,18 @@ class MediaContent(DetailView):
pth = pth[1]
else:
pth = pth[0]
-
-
+
+
# build final path
filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
-
+
if filepth.exists():
# return file
response = FileResponse(open(filepth,'rb'))
return response
else:
return HttpResponseNotFound()
-
+
else:
headers = {
'Content-Type': self.object.content_type,
diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 1eac4c7f..259beea3 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -5,10 +5,16 @@
import os
-from pathlib import Path
-from django.conf import settings
-from copy import copy
+
+from collections import namedtuple
from common.logger import log
+from copy import deepcopy
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from urllib.parse import urlsplit, parse_qs
+
+from django.conf import settings
+from .utils import mkdir_p
import yt_dlp
@@ -21,7 +27,7 @@ _youtubedl_tempdir = getattr(settings, 'YOUTUBE_DL_TEMPDIR', None)
if _youtubedl_tempdir:
_youtubedl_tempdir = str(_youtubedl_tempdir)
_youtubedl_tempdir_path = Path(_youtubedl_tempdir)
- _youtubedl_tempdir_path.mkdir(parents=True, exist_ok=True)
+ mkdir_p(_youtubedl_tempdir_path)
(_youtubedl_tempdir_path / '.ignore').touch(exist_ok=True)
_paths = _defaults.get('paths', {})
_paths.update({ 'temp': _youtubedl_tempdir, })
@@ -37,7 +43,7 @@ class YouTubeError(yt_dlp.utils.DownloadError):
def get_yt_opts():
- opts = copy(_defaults)
+ opts = deepcopy(_defaults)
cookie_file = settings.COOKIES_FILE
if cookie_file.is_file():
cookie_file_path = str(cookie_file.resolve())
@@ -45,6 +51,32 @@ def get_yt_opts():
opts.update({'cookiefile': cookie_file_path})
return opts
+def get_channel_id(url):
+ # yt-dlp --simulate --no-check-formats --playlist-items 1
+ # --print 'pre_process:%(playlist_channel_id,playlist_id,channel_id)s'
+ opts = get_yt_opts()
+ opts.update({
+ 'skip_download': True,
+ 'simulate': True,
+ 'logger': log,
+ 'extract_flat': True, # Change to False to get detailed info
+ 'check_formats': False,
+ 'playlist_items': '1',
+ })
+
+ with yt_dlp.YoutubeDL(opts) as y:
+ try:
+ response = y.extract_info(url, download=False)
+ except yt_dlp.utils.DownloadError as e:
+ raise YouTubeError(f'Failed to extract channel ID for "{url}": {e}') from e
+ else:
+ try:
+ channel_id = response['channel_id']
+ except Exception as e:
+ raise YouTubeError(f'Failed to extract channel ID for "{url}": {e}') from e
+ else:
+ return channel_id
+
def get_channel_image_info(url):
opts = get_yt_opts()
opts.update({
@@ -57,7 +89,7 @@ def get_channel_image_info(url):
with yt_dlp.YoutubeDL(opts) as y:
try:
response = y.extract_info(url, download=False)
-
+
avatar_url = None
banner_url = None
for thumbnail in response['thumbnails']:
@@ -67,7 +99,7 @@ def get_channel_image_info(url):
banner_url = thumbnail['url']
if banner_url != None and avatar_url != None:
break
-
+
return avatar_url, banner_url
except yt_dlp.utils.DownloadError as e:
raise YouTubeError(f'Failed to extract channel info for "{url}": {e}') from e
@@ -81,6 +113,8 @@ def _subscriber_only(msg='', response=None):
return True
if ': Join this channel' in msg:
return True
+ if 'Join this YouTube channel' in msg:
+ return True
else:
# ignore msg entirely
if not isinstance(response, dict):
@@ -179,6 +213,23 @@ def download_media(url, media_format, extension, output_file, info_json,
log.warn(f'[youtube-dl] unknown event: {str(event)}')
hook.download_progress = 0
+
+ default_opts = yt_dlp.parse_options([]).options
+ pp_opts = deepcopy(default_opts)
+ pp_opts.__dict__.update({
+ 'embedthumbnail': embed_thumbnail,
+ 'addmetadata': embed_metadata,
+ 'addchapters': True,
+ 'embed_infojson': False,
+ 'writethumbnail': False,
+ 'force_keyframes_at_cuts': True,
+ 'sponskrub': False,
+ })
+
+ if skip_sponsors:
+ pp_opts.sponsorblock_mark.update('all,-chapter'.split(','))
+ pp_opts.sponsorblock_remove.update(sponsor_categories or {})
+
ytopts = {
'format': media_format,
'merge_output_format': extension,
@@ -192,28 +243,48 @@ def download_media(url, media_format, extension, output_file, info_json,
'writesubtitles': write_subtitles,
'writeautomaticsub': auto_subtitles,
'subtitleslangs': sub_langs.split(','),
- }
- if not sponsor_categories:
- sponsor_categories = []
- sbopt = {
- 'key': 'SponsorBlock',
- 'categories': sponsor_categories
- }
- ffmdopt = {
- 'key': 'FFmpegMetadata',
- 'add_chapters': embed_metadata,
- 'add_metadata': embed_metadata
+ 'writethumbnail': True,
+ 'check_formats': False,
+ 'overwrites': None,
+ 'sleep_interval': 30,
+ 'max_sleep_interval': 600,
+ 'sleep_interval_requests': 30,
}
opts = get_yt_opts()
ytopts['paths'] = opts.get('paths', {})
+ output_dir = os.path.dirname(output_file)
+ temp_dir_parent = output_dir
+ temp_dir_prefix = '.yt_dlp-'
+ if 'temp' in ytopts['paths']:
+ v_key = parse_qs(urlsplit(url).query).get('v').pop()
+ temp_dir_parent = ytopts['paths']['temp']
+ temp_dir_prefix = f'{temp_dir_prefix}{v_key}-'
+ temp_dir = TemporaryDirectory(prefix=temp_dir_prefix,dir=temp_dir_parent)
+ (Path(temp_dir.name) / '.ignore').touch(exist_ok=True)
ytopts['paths'].update({
- 'home': os.path.dirname(output_file),
+ 'home': output_dir,
+ 'temp': temp_dir.name,
})
- if embed_thumbnail:
- ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
- if skip_sponsors:
- ytopts['postprocessors'].append(sbopt)
- ytopts['postprocessors'].append(ffmdopt)
+
+ codec_options = []
+ ofn = ytopts['outtmpl']
+ if 'av1-' in ofn:
+ codec_options = ['-c:v', 'libsvtav1', '-preset', '8', '-crf', '35']
+ elif 'vp9-' in ofn:
+ codec_options = ['-c:v', 'libvpx-vp9', '-b:v', '0', '-crf', '31']
+ ytopts['postprocessor_args'] = opts.get('postprocessor_args', {})
+ set_ffmpeg_codec = not (
+ ytopts['postprocessor_args'] and
+ ytopts['postprocessor_args']['modifychapters+ffmpeg']
+ )
+ if set_ffmpeg_codec and codec_options:
+ ytopts['postprocessor_args'].update({
+ 'modifychapters+ffmpeg': codec_options,
+ })
+
+ # create the post processors list
+ ytopts['postprocessors'] = list(yt_dlp.get_postprocessors(pp_opts))
+
opts.update(ytopts)
with yt_dlp.YoutubeDL(opts) as y:
diff --git a/tubesync/tubesync/gunicorn.py b/tubesync/tubesync/gunicorn.py
index d59c1389..0058fa65 100644
--- a/tubesync/tubesync/gunicorn.py
+++ b/tubesync/tubesync/gunicorn.py
@@ -10,9 +10,10 @@ def get_num_workers():
num_workers = int(os.getenv('GUNICORN_WORKERS', 3))
except ValueError:
num_workers = cpu_workers
- if 0 > num_workers > cpu_workers:
- num_workers = cpu_workers
- return num_workers
+ if 0 < num_workers < cpu_workers:
+ return num_workers
+ else:
+ return cpu_workers
def get_bind():
diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
index e75778b8..1b974cdf 100644
--- a/tubesync/tubesync/local_settings.py.container
+++ b/tubesync/tubesync/local_settings.py.container
@@ -87,6 +87,20 @@ SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'T
SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False
+SHRINK_NEW_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_NEW', 'false').strip().lower()
+SHRINK_NEW_MEDIA_METADATA = ( 'true' == SHRINK_NEW_MEDIA_METADATA_STR )
+SHRINK_OLD_MEDIA_METADATA_STR = os.getenv('TUBESYNC_SHRINK_OLD', 'false').strip().lower()
+SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR )
+
+
+# TUBESYNC_RENAME_ALL_SOURCES: True or False
+RENAME_ALL_SOURCES_STR = os.getenv('TUBESYNC_RENAME_ALL_SOURCES', 'False').strip().lower()
+RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR )
+# TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories
+RENAME_SOURCES_STR = os.getenv('TUBESYNC_RENAME_SOURCES', '')
+RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None
+
+
VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240"))
diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py
index a888e91d..3c350ab3 100644
--- a/tubesync/tubesync/settings.py
+++ b/tubesync/tubesync/settings.py
@@ -177,6 +177,10 @@ COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
+RENAME_ALL_SOURCES = False
+RENAME_SOURCES = None
+
+
try:
from .local_settings import *
except ImportError as e:
diff --git a/tubesync/tubesync/wsgi.py b/tubesync/tubesync/wsgi.py
index 71c61003..74912aef 100644
--- a/tubesync/tubesync/wsgi.py
+++ b/tubesync/tubesync/wsgi.py
@@ -1,5 +1,4 @@
import os
-from urllib.parse import urljoin
from django.core.wsgi import get_wsgi_application
@@ -16,10 +15,9 @@ def application(environ, start_response):
else:
raise Exception(f'DJANGO_URL_PREFIX must end with a /, '
f'got: {DJANGO_URL_PREFIX}')
- if script_name:
- static_url = urljoin(script_name, 'static/')
+ if script_name is not None:
environ['SCRIPT_NAME'] = script_name
path_info = environ['PATH_INFO']
- if path_info.startswith(script_name) and not path_info.startswith(static_url):
+ if path_info.startswith(script_name):
environ['PATH_INFO'] = path_info[len(script_name) - 1:]
return _application(environ, start_response)
diff --git a/tubesync/upgrade_yt-dlp.sh b/tubesync/upgrade_yt-dlp.sh
new file mode 100755
index 00000000..9da6d555
--- /dev/null
+++ b/tubesync/upgrade_yt-dlp.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+warning_message() {
+ cat <&2
+
+pip3() {
+ local pip_runner pip_whl run_whl
+
+ # pipenv
+ pip_runner='/usr/lib/python3/dist-packages/pipenv/patched/pip/__pip-runner__.py'
+ test -s "${pip_runner}" || pip_runner=''
+
+ # python3-pip-whl
+ pip_whl="$(ls -1r /usr/share/python-wheels/pip-*-py3-none-any.whl | head -n 1)"
+ run_whl="${pip_whl}/pip"
+
+ python3 "${pip_runner:-"${run_whl}"}" "$@"
+}
+
+warning_message
+test -n "${TUBESYNC_DEBUG}" || exit 1
+
+# Use the flag added in 23.0.1, if possible.
+# https://github.com/pypa/pip/pull/11780
+break_system_packages='--break-system-packages'
+pip_version="$(pip3 --version | awk '$1 = "pip" { print $2; exit; }')"
+if [[ "${pip_version}" < "23.0.1" ]]; then
+ break_system_packages=''
+fi
+
+pip3 install --upgrade ${break_system_packages} yt-dlp
+
|