Merge pull request #1030 from tcely/patch-6

Refactor models

tubesync/common/json.py

@@ -0,0 +1,16 @@
+from django.core.serializers.json import DjangoJSONEncoder
+
+
+class JSONEncoder(DjangoJSONEncoder):
+    item_separator = ','
+    key_separator = ':'
+
+    def default(self, obj):
+        try:
+            iterable = iter(obj)
+        except TypeError:
+            pass
+        else:
+            return list(iterable)
+        return super().default(obj)
+

tubesync/sync/mediaservers.py

@@ -29,7 +29,7 @@ class MediaServer:
     def make_request_args(self, uri='/', token_header=None, headers={}, token_param=None, params={}):
         base_parts = urlsplit(self.object.url)
         if self.token is None:
-            self.token = self.object.loaded_options['token'] or None
+            self.token = self.object.options['token'] or None
         if token_header and self.token:
             headers.update({token_header: self.token})
         self.headers.update(headers)
@@ -116,7 +116,7 @@ class PlexMediaServer(MediaServer):
         if port < 1 or port > 65535:
             raise ValidationError('Plex Media Server "port" must be between 1 '
                                   'and 65535')
-        options = self.object.loaded_options
+        options = self.object.options
         if 'token' not in options:
             raise ValidationError('Plex Media Server requires a "token"')
         token = options['token'].strip()
@@ -183,7 +183,7 @@ class PlexMediaServer(MediaServer):
 
     def update(self):
         # For each section / library ID pop off a request to refresh it
-        libraries = self.object.loaded_options.get('libraries', '')
+        libraries = self.object.options.get('libraries', '')
         for library_id in libraries.split(','):
             library_id = library_id.strip()
             uri = f'/library/sections/{library_id}/refresh'
@@ -258,7 +258,7 @@ class JellyfinMediaServer(MediaServer):
         except (TypeError, ValueError):
             raise ValidationError('Jellyfin Media Server "port" must be an integer')
 
-        options = self.object.loaded_options
+        options = self.object.options
         if 'token' not in options:
             raise ValidationError('Jellyfin Media Server requires a "token"')
         if 'libraries' not in options:
@@ -302,7 +302,7 @@ class JellyfinMediaServer(MediaServer):
         return True
 
     def update(self):
-        libraries = self.object.loaded_options.get('libraries', '').split(',')
+        libraries = self.object.options.get('libraries', '').split(',')
         for library_id in map(str.strip, libraries):
             uri = f'/Items/{library_id}/Refresh'
             response = self.make_request(uri, method='POST')

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -1,7 +1,7 @@
 # Generated by Django 3.2.18 on 2023-02-14 20:52
 
 from django.db import migrations, models
-import sync.models
+import sync.fields
 
 
 class Migration(migrations.Migration):
@@ -29,6 +29,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='source',
             name='sponsorblock_categories',
-            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
+            field=sync.fields.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
         ),
     ]

tubesync/sync/migrations/0031_metadata_metadataformat.py

@@ -1,7 +1,7 @@
 # Generated by Django 5.1.8 on 2025-04-11 07:36
 
+import common.json
 import django.db.models.deletion
-import sync.models
 import uuid
 from django.db import migrations, models
 
@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
                 ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')),
                 ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')),
                 ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')),
-                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')),
+                ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')),
                 ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')),
             ],
             options={
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
                 ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')),
                 ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')),
                 ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')),
-                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')),
+                ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')),
                 ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')),
             ],
             options={

tubesync/sync/migrations/0031_squashed_metadata_metadataformat.py

@@ -1,7 +1,7 @@
 # Generated by Django 5.1.8 on 2025-04-23 18:10
 
+import common.json
 import django.db.models.deletion
-import sync.models
 import uuid
 from django.db import migrations, models
 
@@ -25,7 +25,7 @@ class Migration(migrations.Migration):
                 ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')),
                 ('uploaded', models.DateTimeField(db_index=True, help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')),
                 ('published', models.DateTimeField(db_index=True, help_text='Date and time the media was published', null=True, verbose_name='published')),
-                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')),
+                ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata object', verbose_name='value')),
                 ('media', models.OneToOneField(help_text='Media the metadata belongs to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='new_metadata', to='sync.media')),
             ],
             options={
@@ -43,7 +43,7 @@ class Migration(migrations.Migration):
                 ('key', models.CharField(blank=True, db_index=True, default='', help_text='Media identifier at the site from which this format is available', max_length=256, verbose_name='key')),
                 ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')),
                 ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')),
-                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')),
+                ('value', models.JSONField(default=dict, encoder=common.json.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')),
                 ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='format', to='sync.metadata')),
             ],
             options={

tubesync/sync/migrations/0033_alter_mediaserver_options_alter_source_source_acodec_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.1.9 on 2025-05-10 06:18
+
+import common.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0032_metadata_transfer'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mediaserver',
+            name='options',
+            field=models.JSONField(encoder=common.json.JSONEncoder, help_text='Options for the media server', null=True, verbose_name='options'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_acodec',
+            field=models.CharField(choices=[('OPUS', 'OPUS'), ('MP4A', 'MP4A')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_vcodec',
+            field=models.CharField(choices=[('AV1', 'AV1'), ('VP9', 'VP9'), ('AVC1', 'AVC1 (H.264)')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'),
+        ),
+    ]

tubesync/sync/models/__init__.py

@@ -0,0 +1,19 @@
+# These are referenced from the migration files
+
+from ._migrations import (
+    get_media_file_path,
+    get_media_thumb_path,
+    media_file_storage,
+)
+
+# The actual model classes
+# The order starts with independent classes
+# then the classes that depend on them follow.
+
+from .media_server import MediaServer
+
+from .source import Source
+from .media import Media
+from .metadata import Metadata
+from .metadata_format import MetadataFormat
+

tubesync/sync/models/_migrations.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+
+
+media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
+
+
+def get_media_file_path(instance, filename):
+    return instance.filepath
+
+
+def get_media_thumb_path(instance, filename):
+    # we don't want to use alternate names for thumb files
+    if instance.thumb:
+        instance.thumb.delete(save=False)
+    fileid = str(instance.uuid).lower()
+    filename = f'{fileid}.jpg'
+    prefix = fileid[:2]
+    return Path('thumbs') / prefix / filename
+

tubesync/sync/models/_private.py

@@ -0,0 +1,12 @@
+from ..choices import Val, YouTube_SourceType
+
+
+_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))
+
+
+def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2):
+    element = nfo.makeelement(label, attrs)
+    element.text = text
+    element.tail = tail + (char * indent)
+    return element
+

tubesync/sync/models/media_server.py

@@ -0,0 +1,86 @@
+from common.json import JSONEncoder
+from django import db
+from django.utils.translation import gettext_lazy as _
+from ..choices import Val, MediaServerType
+
+
+class MediaServer(db.models.Model):
+    '''
+        A remote media server, such as a Plex server.
+    '''
+
+    ICONS = {
+        Val(MediaServerType.JELLYFIN): '<i class="fas fa-server"></i>',
+        Val(MediaServerType.PLEX): '<i class="fas fa-server"></i>',
+    }
+    HANDLERS = MediaServerType.handlers_dict()
+
+    server_type = db.models.CharField(
+        _('server type'),
+        max_length=1,
+        db_index=True,
+        choices=MediaServerType.choices,
+        default=MediaServerType.PLEX,
+        help_text=_('Server type'),
+    )
+    host = db.models.CharField(
+        _('host'),
+        db_index=True,
+        max_length=200,
+        help_text=_('Hostname or IP address of the media server'),
+    )
+    port = db.models.PositiveIntegerField(
+        _('port'),
+        db_index=True,
+        help_text=_('Port number of the media server'),
+    )
+    use_https = db.models.BooleanField(
+        _('use https'),
+        default=False,
+        help_text=_('Connect to the media server over HTTPS'),
+    )
+    verify_https = db.models.BooleanField(
+        _('verify https'),
+        default=True,
+        help_text=_('If connecting over HTTPS, verify the SSL certificate is valid'),
+    )
+    options = db.models.JSONField(
+        _('options'),
+        encoder=JSONEncoder,
+        blank=False,
+        null=True,
+        help_text=_('Options for the media server'),
+    )
+
+    def __str__(self):
+        return f'{self.get_server_type_display()} server at {self.url}'
+
+    class Meta:
+        verbose_name = _('Media Server')
+        verbose_name_plural = _('Media Servers')
+        unique_together = (
+            ('host', 'port'),
+        )
+
+    @property
+    def url(self):
+        scheme = 'https' if self.use_https else 'http'
+        return f'{scheme}://{self.host.strip()}:{self.port}'
+
+    @property
+    def icon(self):
+        return self.ICONS.get(self.server_type)
+
+    @property
+    def handler(self):
+        handler_class = self.HANDLERS.get(self.server_type)
+        return handler_class(self)
+
+    def validate(self):
+        return self.handler.validate()
+
+    def update(self):
+        return self.handler.update()
+
+    def get_help_html(self):
+        return self.handler.HELP

tubesync/sync/models/metadata.py

@@ -0,0 +1,153 @@
+import uuid
+from common.json import JSONEncoder
+from common.timestamp import timestamp_to_datetime
+from common.utils import django_queryset_generator as qs_gen
+from django import db
+from django.utils.translation import gettext_lazy as _
+from .media import Media
+
+
+class Metadata(db.models.Model):
+    '''
+        Metadata for an indexed `Media` item.
+    '''
+    class Meta:
+        db_table = 'sync_media_metadata'
+        verbose_name = _('Metadata about Media')
+        verbose_name_plural = _('Metadata about Media')
+        unique_together = (
+            ('media', 'site', 'key'),
+        )
+        get_latest_by = ["-retrieved", "-created"]
+
+    uuid = db.models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the metadata'),
+    )
+    media = db.models.OneToOneField(
+        Media,
+        # on_delete=models.DO_NOTHING,
+        on_delete=db.models.SET_NULL,
+        related_name='new_metadata',
+        help_text=_('Media the metadata belongs to'),
+        null=True,
+        parent_link=False,
+    )
+    site = db.models.CharField(
+        _('site'),
+        max_length=256,
+        blank=True,
+        db_index=True,
+        null=False,
+        default='Youtube',
+        help_text=_('Site from which the metadata was retrieved'),
+    )
+    key = db.models.CharField(
+        _('key'),
+        max_length=256,
+        blank=True,
+        db_index=True,
+        null=False,
+        default='',
+        help_text=_('Media identifier at the site from which the metadata was retrieved'),
+    )
+    created = db.models.DateTimeField(
+        _('created'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the metadata was created'),
+    )
+    retrieved = db.models.DateTimeField(
+        _('retrieved'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the metadata was retrieved'),
+    )
+    uploaded = db.models.DateTimeField(
+        _('uploaded'),
+        db_index=True,
+        null=True,
+        help_text=_('Date and time the media was uploaded'),
+    )
+    published = db.models.DateTimeField(
+        _('published'),
+        db_index=True,
+        null=True,
+        help_text=_('Date and time the media was published'),
+    )
+    value = db.models.JSONField(
+        _('value'),
+        encoder=JSONEncoder,
+        null=False,
+        default=dict,
+        help_text=_('JSON metadata object'),
+    )
+
+
+    def __str__(self):
+        template = '"{}" from {} at: {}'
+        return template.format(
+            self.key,
+            self.site,
+            self.retrieved.isoformat(timespec='seconds'),
+        )
+
+    @db.transaction.atomic(durable=False)
+    def ingest_formats(self, formats=list(), /):
+        number = 0
+        for number, format in enumerate(formats, start=1):
+            mdf, created = self.format.get_or_create(site=self.site, key=self.key, number=number)
+            mdf.value = format
+            mdf.save()
+        if number > 0:
+            # delete any numbers we did not overwrite or create
+            self.format.filter(site=self.site, key=self.key, number__gt=number).delete()
+
+    @property
+    def with_formats(self):
+        formats = self.format.all().order_by('number')
+        formats_list = [ f.value for f in qs_gen(formats) ]
+        metadata = self.value.copy()
+        metadata.update(dict(formats=formats_list))
+        return metadata
+
+    @db.transaction.atomic(durable=False)
+    def ingest_metadata(self, data):
+        assert isinstance(data, dict), type(data)
+
+        try:
+            self.retrieved = timestamp_to_datetime(
+                self.media.get_metadata_first_value(
+                    'epoch',
+                    arg_dict=data,
+                )
+            ) or self.created
+        except AssertionError:
+            self.retrieved = self.created
+
+        try:
+            self.published = timestamp_to_datetime(
+                self.media.get_metadata_first_value(
+                    ('release_timestamp', 'timestamp',),
+                    arg_dict=data,
+                )
+            ) or self.media.published
+        except AssertionError:
+            self.published = self.media.published
+
+        self.value = data.copy() # try not to have side-effects for the caller
+        formats_key = self.media.get_metadata_field('formats')
+        formats = self.value.pop(formats_key, list())
+        self.uploaded = min(
+            self.published,
+            self.retrieved,
+            self.media.created,
+        )
+        self.save()
+        self.ingest_formats(formats)
+
+        return self.with_formats
+

tubesync/sync/models/metadata_format.py

@@ -0,0 +1,75 @@
+import uuid
+from common.json import JSONEncoder
+from django import db
+from django.utils.translation import gettext_lazy as _
+from .metadata import Metadata
+
+class MetadataFormat(db.models.Model):
+    '''
+        A format from the Metadata for an indexed `Media` item.
+    '''
+    class Meta:
+        db_table = f'{Metadata._meta.db_table}_format'
+        verbose_name = _('Format from Media Metadata')
+        verbose_name_plural = _('Formats from Media Metadata')
+        unique_together = (
+            ('metadata', 'site', 'key', 'number'),
+        )
+        ordering = ['site', 'key', 'number']
+
+    uuid = db.models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the format'),
+    )
+    metadata = db.models.ForeignKey(
+        Metadata,
+        # on_delete=models.DO_NOTHING,
+        on_delete=db.models.CASCADE,
+        related_name='format',
+        help_text=_('Metadata the format belongs to'),
+        null=False,
+    )
+    site = db.models.CharField(
+        _('site'),
+        max_length=256,
+        blank=True,
+        db_index=True,
+        null=False,
+        default='Youtube',
+        help_text=_('Site from which the format is available'),
+    )
+    key = db.models.CharField(
+        _('key'),
+        max_length=256,
+        blank=True,
+        db_index=True,
+        null=False,
+        default='',
+        help_text=_('Media identifier at the site from which this format is available'),
+    )
+    number = db.models.PositiveIntegerField(
+        _('number'),
+        blank=False,
+        null=False,
+        help_text=_('Ordering number for this format'),
+    )
+    value = db.models.JSONField(
+        _('value'),
+        encoder=JSONEncoder,
+        null=False,
+        default=dict,
+        help_text=_('JSON metadata format object'),
+    )
+
+
+    def __str__(self):
+        template = '#{:n} "{}" from {}: {}'
+        return template.format(
+            self.number,
+            self.key,
+            self.site,
+            self.value.get('format') or self.value.get('format_id'),
+        )

tubesync/sync/models/source.py

@@ -0,0 +1,549 @@
+import os
+import re
+import uuid
+from pathlib import Path
+from django import db
+from django.conf import settings
+from django.core.exceptions import SuspiciousOperation
+from django.core.validators import RegexValidator
+from django.utils import timezone
+from django.utils.text import slugify
+from django.utils.translation import gettext_lazy as _
+from ..choices import (Val,
+    SponsorBlock_Category, YouTube_SourceType, IndexSchedule,
+    CapChoices, Fallback, FileExtension, FilterSeconds,
+    SourceResolution, SourceResolutionInteger,
+    YouTube_VideoCodec, YouTube_AudioCodec,
+)
+from ..fields import CommaSepChoiceField
+from ..youtube import (
+    get_media_info as get_youtube_media_info,
+    get_channel_image_info as get_youtube_channel_image_info,
+)
+from ._migrations import media_file_storage
+from ._private import _srctype_dict
+
+
+class Source(db.models.Model):
+    '''
+        A Source is a source of media. Currently, this is either a YouTube channel
+        or a YouTube playlist.
+    '''
+
+    sponsorblock_categories = CommaSepChoiceField(
+        _(''),
+        max_length=128,
+        possible_choices=SponsorBlock_Category.choices,
+        all_choice='all',
+        allow_all=True,
+        all_label='(All Categories)',
+        default='all',
+        help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'),
+    )
+    embed_metadata = db.models.BooleanField(
+        _('embed metadata'),
+        default=False,
+        help_text=_('Embed metadata from source into file'),
+    )
+    embed_thumbnail = db.models.BooleanField(
+        _('embed thumbnail'),
+        default=False,
+        help_text=_('Embed thumbnail into the file'),
+    )
+    enable_sponsorblock = db.models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?'),
+    )
+
+    # Fontawesome icons used for the source on the front end
+    ICONS = _srctype_dict('<i class="fab fa-youtube"></i>')
+
+    # Format to use to display a URL for the source
+    URLS = dict(zip(
+        YouTube_SourceType.values,
+        (
+            'https://www.youtube.com/c/{key}',
+            'https://www.youtube.com/channel/{key}',
+            'https://www.youtube.com/playlist?list={key}',
+        ),
+    ))
+
+    # Format used to create indexable URLs
+    INDEX_URLS = dict(zip(
+        YouTube_SourceType.values,
+        (
+            'https://www.youtube.com/c/{key}/{type}',
+            'https://www.youtube.com/channel/{key}/{type}',
+            'https://www.youtube.com/playlist?list={key}',
+        ),
+    ))
+
+    # Callback functions to get a list of media from the source
+    INDEXERS = _srctype_dict(get_youtube_media_info)
+
+    # Field names to find the media ID used as the key when storing media
+    KEY_FIELD = _srctype_dict('id')
+
+    uuid = db.models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the source'),
+    )
+    created = db.models.DateTimeField(
+        _('created'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the source was created'),
+    )
+    last_crawl = db.models.DateTimeField(
+        _('last crawl'),
+        db_index=True,
+        null=True,
+        blank=True,
+        help_text=_('Date and time the source was last crawled'),
+    )
+    source_type = db.models.CharField(
+        _('source type'),
+        max_length=1,
+        db_index=True,
+        choices=YouTube_SourceType.choices,
+        default=YouTube_SourceType.CHANNEL,
+        help_text=_('Source type'),
+    )
+    key = db.models.CharField(
+        _('key'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Source key, such as exact YouTube channel name or playlist ID'),
+    )
+    name = db.models.CharField(
+        _('name'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Friendly name for the source, used locally in TubeSync only'),
+    )
+    directory = db.models.CharField(
+        _('directory'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Directory name to save the media into'),
+    )
+    media_format = db.models.CharField(
+        _('media format'),
+        max_length=200,
+        default=settings.MEDIA_FORMATSTR_DEFAULT,
+        help_text=_('File format to use for saving files, detailed options at bottom of page.'),
+    )
+    index_schedule = db.models.IntegerField(
+        _('index schedule'),
+        choices=IndexSchedule.choices,
+        db_index=True,
+        default=IndexSchedule.EVERY_24_HOURS,
+        help_text=_('Schedule of how often to index the source for new media'),
+    )
+    download_media = db.models.BooleanField(
+        _('download media'),
+        default=True,
+        help_text=_('Download media from this source, if not selected the source will only be indexed'),
+    )
+    index_videos = db.models.BooleanField(
+        _('index videos'),
+        default=True,
+        help_text=_('Index video media from this source'),
+    )
+    index_streams = db.models.BooleanField(
+        _('index streams'),
+        default=False,
+        help_text=_('Index live stream media from this source'),
+    )
+    download_cap = db.models.IntegerField(
+        _('download cap'),
+        choices=CapChoices.choices,
+        default=CapChoices.CAP_NOCAP,
+        help_text=_('Do not download media older than this capped date'),
+    )
+    delete_old_media = db.models.BooleanField(
+        _('delete old media'),
+        default=False,
+        help_text=_('Delete old media after "days to keep" days?'),
+    )
+    days_to_keep = db.models.PositiveSmallIntegerField(
+        _('days to keep'),
+        default=14,
+        help_text=_(
+            'If "delete old media" is ticked, the number of days after which '
+            'to automatically delete media'
+        ),
+    )
+    filter_text = db.models.CharField(
+        _('filter string'),
+        max_length=200,
+        default='',
+        blank=True,
+        help_text=_('Regex compatible filter string for video titles'),
+    )
+    filter_text_invert = db.models.BooleanField(
+        _('invert filter text matching'),
+        default=False,
+        help_text=_('Invert filter string regex match, skip any matching titles when selected'),
+    )
+    filter_seconds = db.models.PositiveIntegerField(
+        _('filter seconds'),
+        blank=True,
+        null=True,
+        help_text=_('Filter Media based on Min/Max duration. Leave blank or 0 to disable filtering'),
+    )
+    filter_seconds_min = db.models.BooleanField(
+        _('filter seconds min/max'),
+        choices=FilterSeconds.choices,
+        default=Val(FilterSeconds.MIN),
+        help_text=_(
+            'When Filter Seconds is > 0, do we skip on minimum (video shorter than limit) or maximum (video '
+            'greater than maximum) video duration'
+        ),
+    )
+    delete_removed_media = db.models.BooleanField(
+        _('delete removed media'),
+        default=False,
+        help_text=_('Delete media that is no longer on this playlist'),
+    )
+    delete_files_on_disk = db.models.BooleanField(
+        _('delete files on disk'),
+        default=False,
+        help_text=_('Delete files on disk when they are removed from TubeSync'),
+    )
+    source_resolution = db.models.CharField(
+        _('source resolution'),
+        max_length=8,
+        db_index=True,
+        choices=SourceResolution.choices,
+        default=SourceResolution.VIDEO_1080P,
+        help_text=_('Source resolution, desired video resolution to download'),
+    )
+    source_vcodec = db.models.CharField(
+        _('source video codec'),
+        max_length=8,
+        db_index=True,
+        choices=YouTube_VideoCodec.choices,
+        default=YouTube_VideoCodec.VP9,
+        help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)'),
+    )
+    source_acodec = db.models.CharField(
+        _('source audio codec'),
+        max_length=8,
+        db_index=True,
+        choices=YouTube_AudioCodec.choices,
+        default=YouTube_AudioCodec.OPUS,
+        help_text=_('Source audio codec, desired audio encoding format to download'),
+    )
+    prefer_60fps = db.models.BooleanField(
+        _('prefer 60fps'),
+        default=True,
+        help_text=_('Where possible, prefer 60fps media for this source'),
+    )
+    prefer_hdr = db.models.BooleanField(
+        _('prefer hdr'),
+        default=False,
+        help_text=_('Where possible, prefer HDR media for this source'),
+    )
+    fallback = db.models.CharField(
+        _('fallback'),
+        max_length=1,
+        db_index=True,
+        choices=Fallback.choices,
+        default=Fallback.NEXT_BEST_HD,
+        help_text=_('What do do when media in your source resolution and codecs is not available'),
+    )
+    copy_channel_images = db.models.BooleanField(
+        _('copy channel images'),
+        default=False,
+        help_text=_('Copy channel banner and avatar. These may be detected and used by some media servers'),
+    )
+    copy_thumbnails = db.models.BooleanField(
+        _('copy thumbnails'),
+        default=False,
+        help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers'),
+    )
+    write_nfo = db.models.BooleanField(
+        _('write nfo'),
+        default=False,
+        help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers'),
+    )
+    write_json = db.models.BooleanField(
+        _('write json'),
+        default=False,
+        help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers'),
+    )
+    has_failed = db.models.BooleanField(
+        _('has failed'),
+        default=False,
+        help_text=_('Source has failed to index media'),
+    )
+
+    write_subtitles = db.models.BooleanField(
+        _('write subtitles'),
+        default=False,
+        help_text=_('Download video subtitles'),
+    )
+
+    auto_subtitles = db.models.BooleanField(
+        _('accept auto-generated subs'),
+        default=False,
+        help_text=_('Accept auto-generated subtitles'),
+    )
+    sub_langs = db.models.CharField(
+        _('subs langs'),
+        max_length=30,
+        default='en',
+        help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
+        validators=[
+            RegexValidator(
+                regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+",
+                message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat'),
+            ),
+        ],
+    )
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        verbose_name = _('Source')
+        verbose_name_plural = _('Sources')
+
+    @property
+    def icon(self):
+        return self.ICONS.get(self.source_type)
+
+    @property
+    def slugname(self):
+        replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and')
+        return slugify(replaced)[:80]
+
+    def deactivate(self):
+        self.download_media = False
+        self.index_streams = False
+        self.index_videos = False
+        self.index_schedule = IndexSchedule.NEVER
+        self.save(update_fields={
+            'download_media',
+            'index_streams',
+            'index_videos',
+            'index_schedule',
+        })
+
+    @property
+    def is_active(self):
+        active = (
+            self.download_media or
+            self.index_streams or
+            self.index_videos
+        )
+        return self.index_schedule and active
+
+    @property
+    def is_audio(self):
+        return self.source_resolution == SourceResolution.AUDIO.value
+
+    @property
+    def is_playlist(self):
+        return self.source_type == YouTube_SourceType.PLAYLIST.value
+
+    @property
+    def is_video(self):
+        return not self.is_audio
+
+    @property
+    def download_cap_date(self):
+        delta = self.download_cap
+        if delta > 0:
+            return timezone.now() - timezone.timedelta(seconds=delta)
+        else:
+            return False
+
+    @property
+    def days_to_keep_date(self):
+        delta = self.days_to_keep
+        if delta > 0:
+            return timezone.now() - timezone.timedelta(days=delta)
+        else:
+            return False
+
+    @property
+    def extension(self):
+        '''
+            The extension is also used by youtube-dl to set the output container. As
+            it is possible to quite easily pick combinations of codecs and containers
+            which are invalid (e.g. OPUS audio in an MP4 container) just set this for
+            people. All video is set to mkv containers, audio-only is set to m4a or ogg
+            depending on audio codec.
+        '''
+        if self.is_audio:
+            if self.source_acodec == Val(YouTube_AudioCodec.MP4A):
+                return Val(FileExtension.M4A)
+            elif self.source_acodec == Val(YouTube_AudioCodec.OPUS):
+                return Val(FileExtension.OGG)
+            else:
+                raise ValueError('Unable to choose audio extension, uknown acodec')
+        else:
+            return Val(FileExtension.MKV)
+
+    @classmethod
+    def create_url(cls, source_type, key):
+        url = cls.URLS.get(source_type)
+        return url.format(key=key)
+
+    @classmethod
+    def create_index_url(cls, source_type, key, type):
+        url = cls.INDEX_URLS.get(source_type)
+        return url.format(key=key, type=type)
+
+    @property
+    def url(self):
+        return self.__class__.create_url(self.source_type, self.key)
+
+    def get_index_url(self, type):
+        return self.__class__.create_index_url(self.source_type, self.key, type)
+
+    @property
+    def format_summary(self):
+        if self.is_audio:
+            vc = 'none'
+        else:
+            vc = self.source_vcodec
+        ac = self.source_acodec
+        f = ' 60FPS' if self.is_video and self.prefer_60fps else ''
+        h = ' HDR' if self.is_video and self.prefer_hdr else ''
+        return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip()
+
+    @property
+    def directory_path(self):
+        download_dir = Path(media_file_storage.location)
+        return download_dir / self.type_directory_path
+
+    @property
+    def type_directory_path(self):
+        if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX:
+            if self.is_audio:
+                return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
+            else:
+                return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
+        else:
+            return Path(self.directory)
+
+    def make_directory(self):
+        return os.makedirs(self.directory_path, exist_ok=True)
+
+    @property
+    def get_image_url(self):
+        if self.is_playlist:
+            raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.')
+
+        return get_youtube_channel_image_info(self.url)
+
+
+    def directory_exists(self):
+        return (os.path.isdir(self.directory_path) and
+                os.access(self.directory_path, os.W_OK))
+
+    @property
+    def key_field(self):
+        return self.KEY_FIELD.get(self.source_type, '')
+
+    @property
+    def source_resolution_height(self):
+        return SourceResolutionInteger.get(self.source_resolution, 0)
+
+    @property
+    def can_fallback(self):
+        return self.fallback != Val(Fallback.FAIL)
+
+    @property
+    def example_media_format_dict(self):
+        '''
+            Populates a dict with real-ish and some placeholder data for media name
+            format strings. Used for example filenames and media_format validation.
+        '''
+        fmt = []
+        if self.source_resolution:
+            fmt.append(self.source_resolution)
+        if self.source_vcodec:
+            fmt.append(self.source_vcodec.lower())
+        if self.source_acodec:
+            fmt.append(self.source_acodec.lower())
+        if self.prefer_60fps:
+            fmt.append('60fps')
+        if self.prefer_hdr:
+            fmt.append('hdr')
+        now = timezone.now()
+        return {
+            'yyyymmdd': now.strftime('%Y%m%d'),
+            'yyyy_mm_dd': now.strftime('%Y-%m-%d'),
+            'yyyy': now.strftime('%Y'),
+            'mm': now.strftime('%m'),
+            'dd': now.strftime('%d'),
+            'source': self.slugname,
+            'source_full': self.name,
+            'uploader': 'Some Channel Name',
+            'title': 'some-media-title-name',
+            'title_full': 'Some Media Title Name',
+            'key': 'SoMeUnIqUiD',
+            'format': '-'.join(fmt),
+            'playlist_title': 'Some Playlist Title',
+            'video_order': '01',
+            'ext': self.extension,
+            'resolution': self.source_resolution if self.source_resolution else '',
+            'height': '720' if self.source_resolution else '',
+            'width': '1280' if self.source_resolution else '',
+            'vcodec': self.source_vcodec.lower() if self.source_vcodec else '',
+            'acodec': self.source_acodec.lower(),
+            'fps': '24' if self.source_resolution else '',
+            'hdr': 'hdr' if self.source_resolution else ''
+        }
+
+    def get_example_media_format(self):
+        try:
+            return self.media_format.format(**self.example_media_format_dict)
+        except Exception as e:
+            return ''
+
+    def is_regex_match(self, media_item_title):
+        if not self.filter_text:
+            return True
+        return bool(re.search(self.filter_text, media_item_title))
+
+    def get_index(self, type):
+        indexer = self.INDEXERS.get(self.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        days = None
+        if self.download_cap_date:
+            days = timezone.timedelta(seconds=self.download_cap).days
+        response = indexer(self.get_index_url(type=type), days=days)
+        if not isinstance(response, dict):
+            return []
+        entries = response.get('entries', []) 
+        return entries
+
+    def index_media(self):
+        '''
+            Index the media source returning a list of media metadata as dicts.
+        '''
+        entries = list()
+        if self.index_videos:
+            entries += self.get_index('videos')
+        # Playlists do something different that I have yet to figure out
+        if not self.is_playlist:
+            if self.index_streams:
+                entries += self.get_index('streams')
+
+        if settings.MAX_ENTRIES_PROCESSING:
+            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
+        return entries
+

tubesync/sync/templates/sync/mediaserver.html

@@ -28,7 +28,7 @@
         <td class="hide-on-small-only">Verify HTTPS</td>
         <td><span class="hide-on-med-and-up">Verify HTTPS<br></span><strong>{% if mediaserver.verify_https %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
-      {% for name, value in mediaserver.loaded_options.items %}
+      {% for name, value in mediaserver.options.items %}
       <tr title="Unique key of the source, such as the channel name or playlist ID">
         <td class="hide-on-small-only">{{ name|title }}</td>
         <td><span class="hide-on-med-and-up">{{ name|title }}<br></span><strong>{% if name in private_options %}{{ value|truncatechars:6 }} (hidden){% else %}{{ value }}{% endif %}</strong></td>

tubesync/sync/views.py

@@ -1,6 +1,5 @@
 import glob
 import os
-import json
 from base64 import b64decode
 import pathlib
 import sys
@@ -1153,14 +1152,14 @@ class AddMediaServerView(FormView):
     def form_valid(self, form):
         # Assign mandatory fields, bundle other fields into options
         mediaserver = MediaServer(server_type=self.server_type)
-        options = {}
+        options = dict()
         model_fields = [field.name for field in MediaServer._meta.fields]
         for field_name, field_value in form.cleaned_data.items():
             if field_name in model_fields:
                 setattr(mediaserver, field_name, field_value)
             else:
                 options[field_name] = field_value
-        mediaserver.options = json.dumps(options)
+        mediaserver.options = options
         # Test the media server details are valid
         try:
             mediaserver.validate()
@@ -1267,21 +1266,21 @@ class UpdateMediaServerView(FormView, SingleObjectMixin):
         for field in self.object._meta.fields:
             if field.name in self.form_class.declared_fields:
                 initial[field.name] = getattr(self.object, field.name)
-        for option_key, option_val in self.object.loaded_options.items():
+        for option_key, option_val in self.object.options.items():
             if option_key in self.form_class.declared_fields:
                 initial[option_key] = option_val
         return initial
 
     def form_valid(self, form):
         # Assign mandatory fields, bundle other fields into options
-        options = {}
+        options = dict()
         model_fields = [field.name for field in MediaServer._meta.fields]
         for field_name, field_value in form.cleaned_data.items():
             if field_name in model_fields:
                 setattr(self.object, field_name, field_value)
             else:
                 options[field_name] = field_value
-        self.object.options = json.dumps(options)
+        self.object.options = options
         # Test the media server details are valid
         try:
             self.object.validate()

tubesync/tubesync/settings.py

@@ -7,7 +7,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.15.0'
+VERSION = '0.15.1'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []