Merge pull request #523 from FaySmash/tweaked

Add support for indexing Streams

tubesync/sync/migrations/0025_add_video_type_support.py

@@ -0,0 +1,20 @@
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0024_auto_20240717_1535'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='index_videos',
+            field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='index_streams',
+            field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'),
+        ),
+    ]

tubesync/sync/models.py

@@ -165,8 +165,8 @@ class Source(models.Model):
     }
     # Format used to create indexable URLs
     INDEX_URLS = {
-        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}',
-        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
     }
     # Callback functions to get a list of media from the source
@@ -274,6 +274,16 @@ class Source(models.Model):
         default=True,
         help_text=_('Download media from this source, if not selected the source will only be indexed')
     )
+    index_videos = models.BooleanField(
+        _('index videos'),
+        default=False,
+        help_text=_('Index video media from this source')
+    )
+    index_streams = models.BooleanField(
+        _('index streams'),
+        default=False,
+        help_text=_('Index live stream media from this source')
+    )
     download_cap = models.IntegerField(
         _('download cap'),
         choices=CapChoices.choices,
@@ -475,17 +485,16 @@ class Source(models.Model):
         return url.format(key=key)
 
     @classmethod
-    def create_index_url(obj, source_type, key):
+    def create_index_url(obj, source_type, key, type):
         url = obj.INDEX_URLS.get(source_type)
-        return url.format(key=key)
+        return url.format(key=key, type=type)
 
     @property
     def url(self):
         return Source.create_url(self.source_type, self.key)
 
-    @property
+    def get_index_url(self, type):
-    def index_url(self):
+        return Source.create_index_url(self.source_type, self.key, type)
-        return Source.create_index_url(self.source_type, self.key)
 
     @property
     def format_summary(self):
@@ -590,23 +599,32 @@ class Source(models.Model):
             return True
         return bool(re.search(self.filter_text, media_item_title))
 
+    def get_index(self, type):
+        indexer = self.INDEXERS.get(self.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        response = indexer(self.get_index_url(type=type))
+        if not isinstance(response, dict):
+            return []
+        entries = response.get('entries', []) 
+        return entries
+
     def index_media(self):
         '''
             Index the media source returning a list of media metadata as dicts.
         '''
-        indexer = self.INDEXERS.get(self.source_type, None)
+        entries = list()
-        if not callable(indexer):
+        if self.index_videos:
-            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+            entries += self.get_index('videos')
-        response = indexer(self.index_url)
+        # Playlists do something different that I have yet to figure out
-        if not isinstance(response, dict):
+        if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
-            return []
+            if self.index_streams:
-        entries = response.get('entries', [])
+                entries += self.get_index('streams')
 
         if settings.MAX_ENTRIES_PROCESSING:
             entries = entries[:settings.MAX_ENTRIES_PROCESSING]
         return entries
 
-
 def get_media_thumb_path(instance, filename):
     fileid = str(instance.uuid)
     filename = f'{fileid.lower()}.jpg'

tubesync/sync/templates/sync/source.html

@@ -73,6 +73,14 @@
         <td class="hide-on-small-only">Index schedule</td>
         <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
       </tr>
+      <tr title="Index videos from this source">
+        <td class="hide-on-small-only">Index videos?</td>
+        <td><span class="hide-on-med-and-up">Index videos?<br></span><strong>{% if source.index_videos %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      <tr title="Index streams from this source">
+        <td class="hide-on-small-only">Index streams?</td>
+        <td><span class="hide-on-med-and-up">Index streams?<br></span><strong>{% if source.index_streams %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       <tr title="Download media from this source">
         <td class="hide-on-small-only">Download media?</td>
         <td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>

tubesync/sync/views.py

@@ -297,7 +297,7 @@ class ValidateSourceView(FormView):
 class EditSourceMixin:
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min',
-              'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media',
               'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
               'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images',
               'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',