diff --git a/README.md b/README.md index 2ea83c54..c5adc5c5 100644 --- a/README.md +++ b/README.md @@ -265,15 +265,7 @@ and less common features: # Warnings -### 1. Automated file renaming -> [!IMPORTANT] -> Currently, file renaming is not enabled by default. -> Enabling this feature by default is planned in an upcoming release, after `2025-006-01`. -> -> To prevent your installation from scheduling media file renaming tasks, -> you must set [`TUBESYNC_RENAME_ALL_SOURCES=False`](#advanced-configuration) in the environment variables or `RENAME_ALL_SOURCES = False` in [`settings.py`](../1fc0462c11741621350053144ab19cba5f266cb2/tubesync/tubesync/settings.py#L183). - -### 2. Index frequency +### 1. Index frequency It's a good idea to add sources with as long of an index frequency as possible. This is the duration between indexes of the source. An index is when TubeSync checks to see @@ -281,7 +273,7 @@ what videos available on a channel or playlist to find new media. Try and keep t long as possible, up to 24 hours. -### 3. Indexing massive channels +### 2. Indexing massive channels If you add a massive channel (one with several thousand videos) to TubeSync and choose "index every hour" or a similarly short interval; it's entirely possible that your TubeSync install may diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py index 6412ad14..92cb6c8f 100644 --- a/tubesync/sync/choices.py +++ b/tubesync/sync/choices.py @@ -167,6 +167,28 @@ class TaskQueue(models.TextChoices): NET = 'network', _('Networking') +class WeekDay(models.IntegerChoices): + MON = 0, _('Monday') + TUE = 1, _('Tuesday') + WED = 2, _('Wednesday') + THU = 3, _('Thursday') + FRI = 4, _('Friday') + SAT = 5, _('Saturday') + SUN = 6, _('Sunday') + + @classmethod + def get(cls, wdn, /): + return cls[cls.names[wdn]] + + @classmethod + def _from_iso(cls, wdn, /): + return cls[cls.names[wdn - 1]] + + @classmethod + def _to_iso(cls, choice, /): + return 1 + choice.value + + class YouTube_SourceType(models.TextChoices): CHANNEL = 'c', _('YouTube channel') CHANNEL_ID = 'i', _('YouTube channel by ID') diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index e46b740f..4911607b 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -2,13 +2,33 @@ from django import forms, VERSION as DJANGO_VERSION from django.utils.translation import gettext_lazy as _ +from .models import Source + if DJANGO_VERSION[0:3] < (5, 0, 0): _assume_scheme = dict() else: # Silence RemovedInDjango60Warning _assume_scheme = dict(assume_scheme='http') - + +SourceForm = forms.modelform_factory( + Source, + # manual ordering + fields = ( + 'source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', + 'media_format', 'target_schedule', 'index_schedule', 'index_videos', 'index_streams', 'download_media', + 'download_cap', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', + 'prefer_60fps', 'prefer_hdr', 'fallback', 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images', + 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', + 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs', + ), + widgets = { + 'target_schedule': forms.DateTimeInput( + attrs={'type': 'datetime-local'}, + ), + }, +) + class ValidateSourceForm(forms.Form): source_type = forms.CharField( diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index 318a5b27..ae38a464 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -29,7 +29,7 @@ class Command(BaseCommand): index_source_task( str(source.pk), repeat=source.index_schedule, - schedule=source.index_schedule, + schedule=source.task_run_at_dt, verbose_name=verbose_name.format(source.name), ) # This also chains down to call each Media objects .save() as well diff --git a/tubesync/sync/migrations/0034_source_target_schedule_and_more.py b/tubesync/sync/migrations/0034_source_target_schedule_and_more.py new file mode 100644 index 00000000..a2869557 --- /dev/null +++ b/tubesync/sync/migrations/0034_source_target_schedule_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.1 on 2025-05-26 04:43 + +import django.utils.timezone +import sync.fields +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0033_alter_mediaserver_options_alter_source_source_acodec_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='target_schedule', + field=models.DateTimeField( + blank=True, db_index=True, default=django.utils.timezone.now, + help_text='Date and time when the task to index the source should begin', + verbose_name='target schedule', + ), + ), + migrations.AlterField( + model_name='source', + name='sponsorblock_categories', + field=sync.fields.CommaSepChoiceField( + all_choice='all', all_label='(All Categories)', allow_all=True, default='all', + help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.', + max_length=128, possible_choices=[ + ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), + ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), + ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'), + ], + verbose_name='removed categories', + ), + ), + ] + diff --git a/tubesync/sync/models/metadata.py b/tubesync/sync/models/metadata.py index 17d214fb..6f0f51ed 100644 --- a/tubesync/sync/models/metadata.py +++ b/tubesync/sync/models/metadata.py @@ -134,9 +134,9 @@ class Metadata(db.models.Model): ('release_timestamp', 'timestamp',), arg_dict=data, ) - ) or self.media.published + ) or self.media.published or self.retrieved except AssertionError: - self.published = self.media.published + self.published = self.media.published or self.retrieved self.value = data.copy() # try not to have side-effects for the caller formats_key = self.media.get_metadata_field('formats') diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 07ade020..a594063e 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -31,16 +31,6 @@ class Source(db.models.Model): or a YouTube playlist. ''' - sponsorblock_categories = CommaSepChoiceField( - _(''), - max_length=128, - possible_choices=SponsorBlock_Category.choices, - all_choice='all', - allow_all=True, - all_label='(All Categories)', - default='all', - help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'), - ) embed_metadata = db.models.BooleanField( _('embed metadata'), default=False, @@ -51,11 +41,6 @@ class Source(db.models.Model): default=False, help_text=_('Embed thumbnail into the file'), ) - enable_sponsorblock = db.models.BooleanField( - _('enable sponsorblock'), - default=True, - help_text=_('Use SponsorBlock?'), - ) # Fontawesome icons used for the source on the front end ICONS = _srctype_dict('') @@ -141,6 +126,13 @@ class Source(db.models.Model): default=settings.MEDIA_FORMATSTR_DEFAULT, help_text=_('File format to use for saving files, detailed options at bottom of page.'), ) + target_schedule = db.models.DateTimeField( + _('target schedule'), + blank=True, + db_index=True, + default=timezone.now, + help_text=_('Date and time when the task to index the source should begin'), + ) index_schedule = db.models.IntegerField( _('index schedule'), choices=IndexSchedule.choices, @@ -310,6 +302,21 @@ class Source(db.models.Model): ), ], ) + enable_sponsorblock = db.models.BooleanField( + _('enable sponsorblock'), + default=True, + help_text=_('Use SponsorBlock?'), + ) + sponsorblock_categories = CommaSepChoiceField( + _('removed categories'), + max_length=128, + possible_choices=SponsorBlock_Category.choices, + all_choice='all', + allow_all=True, + all_label='(All Categories)', + default='all', + help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'), + ) def __str__(self): return self.name @@ -376,6 +383,39 @@ class Source(db.models.Model): else: return False + @property + def task_run_at_dt(self): + now = timezone.now() + when = now.replace(minute=0, second=0, microsecond=0) + + def advance_hour(arg_dt, target_hour, /): + delta_hours = ((24 + target_hour) - arg_dt.hour) % 24 + return arg_dt + timezone.timedelta(hours=delta_hours) + + def advance_day(arg_dt, target_weekday, /): + delta_days = ((7 + target_weekday) - arg_dt.weekday) % 7 + return arg_dt + timezone.timedelta(days=delta_days) + + if self.target_schedule is None: + self.target_schedule = when + if Val(IndexSchedule.EVERY_24_HOURS) > self.index_schedule: + self.target_schedule = now + timezone.timedelta( + seconds=1+self.index_schedule, + ) + elif Val(IndexSchedule.EVERY_7_DAYS) > self.index_schedule: + self.target_schedule = advance_hour( + when.replace(hour=1+when.hour), + self.target_schedule.hour, + ) + + if now < self.target_schedule: + return self.target_schedule + + when = advance_hour(when, self.target_schedule.hour) + when = advance_day(when, self.target_schedule.weekday) + self.target_schedule = when + return when + @property def extension(self): ''' diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 69254146..d68a082f 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -95,7 +95,7 @@ def source_pre_save(sender, instance, **kwargs): index_source_task( str(instance.pk), repeat=instance.index_schedule, - schedule=instance.index_schedule, + schedule=instance.task_run_at_dt, verbose_name=verbose_name.format(instance.name), ) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index bf5e43ed..c5903bb0 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -319,6 +319,8 @@ def index_source_task(source_id): # An inactive Source would return an empty list for videos anyway if not source.is_active: return + # update the target schedule column + source.task_run_at_dt # Reset any errors # TODO: determine if this affects anything source.has_failed = False diff --git a/tubesync/sync/templates/sync/dashboard.html b/tubesync/sync/templates/sync/dashboard.html index 23e1cdb2..af342800 100644 --- a/tubesync/sync/templates/sync/dashboard.html +++ b/tubesync/sync/templates/sync/dashboard.html @@ -99,18 +99,6 @@ -
-
-

Warnings

-
- An upcoming release, after 2025-006-01, will introduce automated file renaming.
- To prevent this change from taking effect, you can set an environment variable before that date.
- See the GitHub README - for more details or ask questions using - issue #785.
-
-
-

Runtime information

diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 19c195f5..f926bbdc 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -93,6 +93,10 @@ Last crawl Last crawl
{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H:i:s' }}{% else %}Never{% endif %} + + Target schedule + Target schedule
{% if source.target_schedule %}{{ source.target_schedule|date:'l, h:00 A (c)' }}{% else %}Not set{% endif %} + Source resolution Source resolution
{{ source.get_source_resolution_display }} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 093cad3f..5f13877f 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -25,7 +25,7 @@ from background_task.models import Task, CompletedTask from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, - ConfirmDeleteMediaServerForm) + ConfirmDeleteMediaServerForm, SourceForm) from .utils import validate_url, delete_file, multi_key_sort, mkdir_p from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, @@ -276,15 +276,7 @@ class ValidateSourceView(FormView): class EditSourceMixin: model = Source - # manual ordering - fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', - 'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media', - 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', - 'prefer_60fps', 'prefer_hdr', 'fallback', - 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images', - 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', - 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', - 'auto_subtitles', 'sub_langs') + form_class = SourceForm errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' @@ -357,6 +349,9 @@ class AddSourceView(EditSourceMixin, CreateView): def get_initial(self): initial = super().get_initial() + initial['target_schedule'] = timezone.now().replace( + second=0, microsecond=0, + ) for k, v in self.prepopulated_data.items(): initial[k] = v return initial @@ -403,6 +398,12 @@ class UpdateSourceView(EditSourceMixin, UpdateView): template_name = 'sync/source-update.html' + def get_initial(self): + initial = super().get_initial() + when = getattr(self.object, 'target_schedule') or timezone.now() + initial['target_schedule'] = when.replace(second=0, microsecond=0) + return initial + def get_success_url(self): url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk}) return append_uri_params(url, {'message': 'source-updated'}) @@ -1003,6 +1004,7 @@ class ResetTasks(FormView): index_source_task( str(source.pk), repeat=source.index_schedule, + schedule=source.task_run_at_dt, verbose_name=verbose_name.format(source.name) ) # This also chains down to call each Media objects .save() as well diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 8b61692b..132f3c40 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -108,11 +108,11 @@ SHRINK_OLD_MEDIA_METADATA = ( 'true' == SHRINK_OLD_MEDIA_METADATA_STR.strip().lo # TUBESYNC_RENAME_ALL_SOURCES: True or False -RENAME_ALL_SOURCES_STR = getenv('TUBESYNC_RENAME_ALL_SOURCES', False) +RENAME_ALL_SOURCES_STR = getenv('TUBESYNC_RENAME_ALL_SOURCES', True) RENAME_ALL_SOURCES = ( 'true' == RENAME_ALL_SOURCES_STR.strip().lower() ) # TUBESYNC_RENAME_SOURCES: A comma-separated list of Source directories RENAME_SOURCES_STR = getenv('TUBESYNC_RENAME_SOURCES') -RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else None +RENAME_SOURCES = RENAME_SOURCES_STR.split(',') if RENAME_SOURCES_STR else list() VIDEO_HEIGHT_CUTOFF = getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", 240, integer=True) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a087da1a..938f7ecd 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -199,8 +199,8 @@ COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}' -RENAME_ALL_SOURCES = False -RENAME_SOURCES = None +RENAME_ALL_SOURCES = True +RENAME_SOURCES = list() # WARNING WARNING WARNING