diff --git a/app/sync/migrations/0016_auto_20201208_0518.py b/app/sync/migrations/0016_auto_20201208_0518.py new file mode 100644 index 00000000..b9604cdf --- /dev/null +++ b/app/sync/migrations/0016_auto_20201208_0518.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-08 05:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0015_auto_20201207_0744'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='fallback', + field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'), + ), + ] diff --git a/app/sync/models.py b/app/sync/models.py index 4de5ce77..32de399a 100644 --- a/app/sync/models.py +++ b/app/sync/models.py @@ -72,13 +72,13 @@ class Source(models.Model): ) FALLBACK_FAIL = 'f' - FALLBACK_NEXT_SD = 's' - FALLBACK_NEXT_HD = 'h' - FALLBACKS = (FALLBACK_FAIL, FALLBACK_NEXT_SD, FALLBACK_NEXT_HD) + FALLBACK_NEXT_BEST = 'n' + FALLBACK_NEXT_BEST_HD = 'h' + FALLBACKS = (FALLBACK_FAIL, FALLBACK_NEXT_BEST, FALLBACK_NEXT_BEST_HD) FALLBACK_CHOICES = ( (FALLBACK_FAIL, _('Fail, do not download any media')), - (FALLBACK_NEXT_SD, _('Get next best SD media or codec instead')), - (FALLBACK_NEXT_HD, _('Get next best HD media or codec instead')), + (FALLBACK_NEXT_BEST, _('Get next best resolution or codec instead')), + (FALLBACK_NEXT_BEST_HD, _('Get next best resolution but at least HD')) ) # Fontawesome icons used for the source on the front end @@ -218,7 +218,7 @@ class Source(models.Model): max_length=1, db_index=True, choices=FALLBACK_CHOICES, - default=FALLBACK_NEXT_HD, + default=FALLBACK_NEXT_BEST_HD, help_text=_('What do do when media in your source resolution and codecs is not available') ) has_failed = models.BooleanField( @@ -300,6 +300,10 @@ class Source(models.Model): def source_resolution_height(self): return self.RESOLUTION_MAP.get(self.source_resolution, 0) + @property + def can_fallback(self): + return self.fallback != self.FALLBACK_FAIL + def index_media(self): ''' Index the media source returning a list of media metadata as dicts. @@ -494,94 +498,237 @@ class Media(models.Model): fields = self.METADATA_FIELDS.get(field, {}) return fields.get(self.source.source_type, '') + def iter_formats(self): + for fmt in self.formats: + yield parse_media_format(fmt) + def get_best_combined_format(self): ''' Attempts to see if there is a single, combined audio and video format that exactly matches the source requirements. This is used over separate audio - and video formats if possible. - - single format structure = { - 'format_id': '22', - 'url': '... long url ...', - 'player_url': None, - 'ext': 'mp4', - 'width': 1280, - 'height': 720, - 'acodec': 'mp4a.40.2', - 'abr': 192, - 'vcodec': 'avc1.64001F', - 'asr': 44100, - 'filesize': None, - 'format_note': '720p', - 'fps': 30, - 'tbr': 1571.695, - 'format': '22 - 1280x720 (720p)', - 'protocol': 'https', - 'http_headers': {... dict of headers ...} - } - - or for hdr = { - 'format_id': '336', - 'url': '... long url ...', - 'player_url': None, - 'asr': None, - 'filesize': 312014985, - 'format_note': '1440p60 HDR', - 'fps': 60, - 'height': 1440, - 'tbr': 16900.587, - 'width': 2560, - 'ext': 'webm', - 'vcodec': 'vp9.2', - 'acodec': 'none', - 'downloader_options': {'http_chunk_size': 10485760}, - 'format': '336 - 2560x1440 (1440p60 HDR)', - 'protocol': 'https', - 'http_headers': {... dict of headers ...} - } - + and video formats if possible. Combined formats are the easiest to check + for as they must exactly match the source profile be be valid. ''' - candidates = [] - for fmt in self.formats: - parsed_fmt = parse_media_format(fmt) + for fmt in self.iter_formats(): # Check height matches - print(self.source.source_resolution_height, parsed_fmt['height']) - if self.source.source_resolution_height != parsed_fmt['height']: - print() + if self.source.source_resolution.strip().upper() != fmt['format']: continue - print('height OK') # Check the video codec matches - print(self.source.source_vcodec, parsed_fmt['vcodec']) - if self.source.source_vcodec != parsed_fmt['vcodec']: - print() + if self.source.source_vcodec != fmt['vcodec']: continue - print('vcodec OK') # Check the audio codec matches - print(self.source.source_acodec, parsed_fmt['acodec']) - if self.source.source_acodec != parsed_fmt['acodec']: - print() + if self.source.source_acodec != fmt['acodec']: continue - print('acodec OK') - # All OK so far... - candidates.append(parsed_fmt) - print() - for c in candidates: - print(c) - return 'combined' + # if the source prefers 60fps, check for it + if self.source.prefer_60fps: + if not fmt['is_60fps']: + continue + # If the source prefers HDR, check for it + if self.source.prefer_hdr: + if not fmt['is_hdr']: + continue + # If we reach here, we have a combined match! + return True, fmt['id'] + return False, False def get_best_audio_format(self): ''' Finds the best match for the source required audio format. If the source has a 'fallback' of fail this can return no match. ''' - return 'audio' + # Order all audio-only formats by bitrate + audio_formats = [] + for fmt in self.iter_formats(): + # If the format has a video stream, skip it + if fmt['vcodec']: + continue + audio_formats.append(fmt) + audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr']))) + if not audio_formats: + # Media has no audio formats at all + return False, False + # Find the highest bitrate audio format with a matching codec + for fmt in audio_formats: + if self.source.source_acodec == fmt['acodec']: + # Matched! + return True, fmt['id'] + # No codecs matched + if self.source.can_fallback: + # Can fallback, find the next highest bitrate non-matching codec + return False, audio_formats[0] + else: + # Can't fallback + return False, False + def get_best_video_format(self): ''' Finds the best match for the source required video format. If the source - has a 'fallback' of fail this can return no match. + has a 'fallback' of fail this can return no match. Resolution is treated + as the most important factor to match. ''' - return 'video' + min_height = getattr(settings, 'VIDEO_HEIGHT_CUTOFF', 360) + fallback_hd_cutoff = getattr(settings, 'VIDEO_HEIGHT_IS_HD', 500) + # Filter video-only formats by resolution that matches the source + video_formats = [] + for fmt in self.iter_formats(): + # If the format has an audio stream, skip it + if fmt['acodec']: + continue + if self.source.source_resolution.strip().upper() == fmt['format']: + video_formats.append(fmt) + # Check we matched some streams + if not video_formats: + # No streams match the requested resolution, see if we can fallback + if self.source.can_fallback: + # Find the next-best format matches by height + for fmt in self.iter_formats(): + # If the format has an audio stream, skip it + if fmt['acodec']: + continue + if (fmt['height'] <= self.source.source_resolution_height and + fmt['height'] >= min_height): + video_formats.append(fmt) + else: + # Can't fallback + return False, False + video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height']))) + if not video_formats: + # Still no matches + return False, False + exact_match, best_match = None, None + # Of our filtered video formats, check for resolution + codec + hdr + fps match + if self.source.prefer_60fps and self.source.prefer_hdr: + for fmt in video_formats: + # Check for an exact match + if (self.source.source_resolution.strip().upper() == fmt['format'] and + self.source.source_vcodec == fmt['vcodec'] and + fmt['is_hdr'] and + fmt['is_60fps']): + # Exact match + exact_match, best_match = True, fmt + break + if self.source.can_fallback: + if not best_match: + for fmt in video_formats: + # Check for a codec, hdr and fps match but drop the resolution + if (self.source.source_vcodec == fmt['vcodec'] and + fmt['is_hdr'] and fmt['is_60fps']): + # Close match + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for hdr and fps match but drop the resolution and codec + if fmt['is_hdr'] and fmt['is_60fps']: + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for fps match but drop the resolution and codec and hdr + if fmt['is_hdr'] and fmt['is_60fps']: + exact_match, best_match = False, fmt + break + if not best_match: + # Match the highest resolution + exact_match, best_match = False, video_formats[0] + # Check for resolution + codec + fps match + if self.source.prefer_60fps and not self.source.prefer_hdr: + for fmt in video_formats: + # Check for an exact match + if (self.source.source_resolution.strip().upper() == fmt['format'] and + self.source.source_vcodec == fmt['vcodec'] and + fmt['is_60fps']): + # Exact match + exact_match, best_match = True, fmt + break + if self.source.can_fallback: + if not best_match: + for fmt in video_formats: + # Check for a codec and fps match but drop the resolution + if (self.source.source_vcodec == fmt['vcodec'] and + fmt['is_60fps']): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for an fps match but drop the resolution and codec + if fmt['is_60fps']: + exact_match, best_match = False, fmt + break + if not best_match: + # Match the highest resolution + exact_match, best_match = False, video_formats[0] + # Check for resolution + codec + hdr + if self.source.prefer_hdr and not self.source.prefer_60fps: + for fmt in video_formats: + # Check for an exact match + if (self.source.source_resolution.strip().upper() == fmt['format'] and + self.source.source_vcodec == fmt['vcodec'] and + fmt['is_hdr']): + # Exact match + exact_match, best_match = True, fmt + break + if self.source.can_fallback: + if not best_match: + for fmt in video_formats: + # Check for a codec and hdr match but drop the resolution + if (self.source.source_vcodec == fmt['vcodec'] and + fmt['is_hdr']): + exact_match, best_match = True, fmt + break + if not best_match: + for fmt in video_formats: + # Check for an hdr match but drop the resolution and codec + if fmt['is_hdr']: + exact_match, best_match = False, fmt + break + if not best_match: + # Match the highest resolution + exact_match, best_match = False, video_formats[0] + # check for resolution + codec + if not self.source.prefer_hdr and not self.source.prefer_60fps: + for fmt in video_formats: + # Check for an exact match + if (self.source.source_resolution.strip().upper() == fmt['format'] and + self.source.source_vcodec == fmt['vcodec'] and + not fmt['is_60fps']): + # Exact match + exact_match, best_match = True, fmt + break + if self.source.can_fallback: + if not best_match: + for fmt in video_formats: + # Check for a codec match without 60fps and drop the resolution + if (self.source.source_vcodec == fmt['vcodec'] and + not fmt['is_60fps']): + exact_match, best_match = False, fmt + break + if not best_match: + for fmt in video_formats: + # Check for a codec match but drop the resolution + if self.source.source_vcodec == fmt['vcodec']: + # Close match + exact_match, best_match = False, fmt + break + if not best_match: + # Match the highest resolution + exact_match, best_match = False, video_formats[0] + # See if we found a match + if best_match: + # Final check to see if the match we found was good enough + if exact_match: + return True, best_match['id'] + elif self.source.can_fallback: + # Allow the fallback if it meets requirements + if (self.source.fallback == self.source.FALLBACK_NEXT_BEST_HD and + best_match['height'] >= fallback_hd_cutoff): + return False, best_match['id'] + elif self.source.fallback == self.source.FALLBACK_NEXT_BEST: + return False, best_match['id'] + # Nope, failed to find match + return False, False + def get_format_str(self): ''' diff --git a/app/sync/templates/sync/media-item.html b/app/sync/templates/sync/media-item.html index d01fa71b..f74926d0 100644 --- a/app/sync/templates/sync/media-item.html +++ b/app/sync/templates/sync/media-item.html @@ -33,6 +33,10 @@ Desired format Desired format
{{ media.source.format_summary }} + + Fallback + Fallback
{{ media.source.get_fallback_display }} + Downloaded Downloaded
{% if media.downloaded %}{% else %}{% endif %} @@ -48,11 +52,11 @@ - Best match - Best match
- audio: {{ media.get_best_audio_format }}
- video: {{ media.get_best_video_format }}
- combo: {{ media.get_best_combined_format }} + Matched formats + Matched formats
+ Combined: {% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}
+ Audio: {% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}
+ Video: {% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %} diff --git a/app/sync/templates/sync/source.html b/app/sync/templates/sync/source.html index 8c98d920..9f831ed1 100644 --- a/app/sync/templates/sync/source.html +++ b/app/sync/templates/sync/source.html @@ -18,6 +18,7 @@ View tasks linked to this source +{% include 'infobox.html' with message=message %} {% if source.has_failed %}{% include 'errorbox.html' with message='This source has encountered permanent failures listed at the bottom of this page, check its settings' %}{% endif %}
diff --git a/app/sync/utils.py b/app/sync/utils.py index 4bc0e47f..f202fe10 100644 --- a/app/sync/utils.py +++ b/app/sync/utils.py @@ -142,11 +142,21 @@ def parse_media_format(format_dict): acodec = None if acodec == 'NONE': acodec = None + try: + fps = int(format_dict.get('fps', 0)) + except (ValueError, TypeError): + fps = 0 + format_full = format_dict.get('format_note', '').strip().upper() + format_str = format_full[:-2] if format_full.endswith('60') else format_full return { 'id': format_dict.get('format_id', ''), + 'format': format_str, + 'format_verbose': format_dict.get('format', ''), 'height': format_dict.get('height', 0), - 'is_60fps': format_dict.get('fps', 0) == 60, - 'is_hdr': 'HDR' in format_dict.get('format', '').upper(), 'vcodec': vcodec, + 'vbr': format_dict.get('tbr', 0), 'acodec': acodec, + 'abr': format_dict.get('abr', 0), + 'is_60fps': fps > 50, + 'is_hdr': 'HDR' in format_dict.get('format', '').upper(), } diff --git a/app/sync/views.py b/app/sync/views.py index 24bb5a00..1949b7be 100644 --- a/app/sync/views.py +++ b/app/sync/views.py @@ -41,9 +41,7 @@ class SourcesView(ListView): context_object_name = 'sources' paginate_by = settings.SOURCES_PER_PAGE messages = { - 'source-created': _('Your new source has been added'), 'source-deleted': _('Your selected source has been deleted.'), - 'source-updated': _('Your selected source has been updated.'), } def __init__(self, *args, **kwargs): @@ -235,7 +233,7 @@ class AddSourceView(CreateView): return initial def get_success_url(self): - url = reverse_lazy('sync:sources') + url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk}) return append_uri_params(url, {'message': 'source-created'}) @@ -243,9 +241,23 @@ class SourceView(DetailView): template_name = 'sync/source.html' model = Source + messages = { + 'source-created': _('Your new source has been created'), + 'source-updated': _('Your source has been updated.'), + } + + def __init__(self, *args, **kwargs): + self.message = None + super().__init__(*args, **kwargs) + + def dispatch(self, request, *args, **kwargs): + message_key = request.GET.get('message', '') + self.message = self.messages.get(message_key, '') + return super().dispatch(request, *args, **kwargs) def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) + data['message'] = self.message data['errors'] = [] for error in get_source_completed_tasks(self.object.pk, only_errors=True): error_message = get_error_message(error) @@ -264,7 +276,7 @@ class UpdateSourceView(UpdateView): 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback') def get_success_url(self): - url = reverse_lazy('sync:sources') + url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk}) return append_uri_params(url, {'message': 'source-updated'}) @@ -366,6 +378,19 @@ class MediaItemView(DetailView): template_name = 'sync/media-item.html' model = Media + def get_context_data(self, *args, **kwargs): + data = super().get_context_data(*args, **kwargs) + combined_exact, combined_format = self.object.get_best_combined_format() + audio_exact, audio_format = self.object.get_best_audio_format() + video_exact, video_format = self.object.get_best_video_format() + data['combined_exact'] = combined_exact + data['combined_format'] = combined_format + data['audio_exact'] = audio_exact + data['audio_format'] = audio_format + data['video_exact'] = video_exact + data['video_format'] = video_format + return data + class TasksView(ListView): ''' diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py index 57c660a2..97560ee1 100644 --- a/app/tubesync/settings.py +++ b/app/tubesync/settings.py @@ -130,6 +130,10 @@ MEDIA_THUMBNAIL_WIDTH = 430 # Width in pixels to resize thumbnai MEDIA_THUMBNAIL_HEIGHT = 240 # Height in pixels to resize thumbnails to +VIDEO_HEIGHT_CUTOFF = 360 # Smallest resolution in pixels permitted to download +VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' + + YOUTUBE_DEFAULTS = { 'no_color': True, # Do not use colours in output 'age_limit': 99, # 'Age in years' to spoof