diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 12753ae8..9aa61d04 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -35,7 +35,7 @@
-

{{ errors|length|intcomma }} Error{{ errors|length|pluralize }}

+

{{ total_errors|intcomma }} Total Error{{ total_errors|pluralize }} ({{ errors|length|intcomma }} on this page)

Tasks which generated an error are shown here. Tasks are retried a couple of times, so if there was an intermittent error such as a download got interrupted @@ -49,14 +49,14 @@ Task will be retried at {{ task.run_at|date:'Y-m-d H:i:s' }} {% empty %} - There are no tasks with errors. + There are no tasks with errors on this page. {% endfor %}

-

{{ total_scheduled|intcomma }} Scheduled

+

{{ total_scheduled|intcomma }} Scheduled ({{ scheduled|length|intcomma }} on this page)

Tasks which are scheduled to run in the future or are waiting in a queue to be processed. They can be waiting for an available worker to run immediately, or @@ -70,7 +70,7 @@ Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }}{% endif %} {% empty %} - There are no scheduled tasks. + There are no scheduled tasks on this page. {% endfor %}

diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 9f599672..917a9531 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -2,7 +2,7 @@ import os import re import math from copy import deepcopy -from operator import itemgetter +from operator import attrgetter, itemgetter from pathlib import Path from tempfile import NamedTemporaryFile import requests @@ -179,10 +179,16 @@ def seconds_to_timestr(seconds): return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) -def multi_key_sort(sort_dict, specs, use_reversed=False): - result = list(sort_dict) +def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None): + result = list(iterable) + if key_func is None: + # itemgetter is the default + if item or not (item or attr): + key_func = itemgetter + elif attr: + key_func = attrgetter for key, reverse in reversed(specs): - result = sorted(result, key=itemgetter(key), reverse=reverse) + result.sort(key=key_func(key), reverse=reverse) if use_reversed: return list(reversed(result)) return result diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 66c17595..cc721ffb 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -27,7 +27,7 @@ from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ConfirmDeleteMediaServerForm) -from .utils import validate_url, delete_file +from .utils import validate_url, delete_file, multi_key_sort from .tasks import (map_task_to_instance, get_error_message, get_source_completed_tasks, get_media_download_task, delete_task_by_media, index_source_task) @@ -782,24 +782,41 @@ class TasksView(ListView): prefix = '-' if 'ASC' != order else '' _priority = f'{prefix}priority' return qs.order_by( - 'run_at', _priority, + 'run_at', ) def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) now = timezone.now() qs = Task.objects.all() + errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True) + running_qs = qs.filter(locked_by__isnull=False) + scheduled_qs = qs.filter(locked_by__isnull=True) # Add to context data from ListView data['message'] = self.message data['source'] = self.filter_source - data['running'] = [] - data['errors'] = [] - data['scheduled'] = [] - data['total_scheduled'] = qs.filter(locked_at__isnull=True).count() + data['running'] = list() + data['errors'] = list() + data['total_errors'] = errors_qs.count() + data['scheduled'] = list() + data['total_scheduled'] = scheduled_qs.count() - for task in qs.filter(locked_at__isnull=False): + def add_to_task(task): + obj, url = map_task_to_instance(task) + if not obj: + return False + setattr(task, 'instance', obj) + setattr(task, 'url', url) + setattr(task, 'run_now', task.run_at < now) + if task.has_error(): + error_message = get_error_message(task) + setattr(task, 'error_message', error_message) + return 'error' + return True + + for task in running_qs: # There was broken logic in `Task.objects.locked()`, work around it. # With that broken logic, the tasks never resume properly. # This check unlocks the tasks without a running process. @@ -807,31 +824,27 @@ class TasksView(ListView): # - `True`: locked and PID exists # - `False`: locked and PID does not exist # - `None`: not `locked_by`, so there was no PID to check - if task.locked_by_pid_running() is False: + locked_by_pid_running = task.locked_by_pid_running() + if locked_by_pid_running is False: task.locked_by = None # do not wait for the task to expire task.locked_at = None task.save() - obj, url = map_task_to_instance(task) - if not obj: - # Orphaned task, ignore it (it will be deleted when it fires) - continue - setattr(task, 'instance', obj) - setattr(task, 'url', url) - setattr(task, 'run_now', task.run_at < now) - if task.locked_by_pid_running(): + if locked_by_pid_running and add_to_task(task): data['running'].append(task) - elif task.has_error(): - error_message = get_error_message(task) - setattr(task, 'error_message', error_message) - data['errors'].append(task) - else: - data['scheduled'].append(task) + + # show all the errors when they fit on one page + if (data['total_errors'] + len(data['running'])) < self.paginate_by: + for task in errors_qs: + if task in data['running']: + continue + mapped = add_to_task(task) + if 'error' == mapped: + data['errors'].append(task) + elif mapped: + data['scheduled'].append(task) for task in data['tasks']: - obj, url = map_task_to_instance(task) - if not obj: - continue already_added = ( task in data['running'] or task in data['errors'] or @@ -839,10 +852,24 @@ class TasksView(ListView): ) if already_added: continue - setattr(task, 'instance', obj) - setattr(task, 'url', url) - setattr(task, 'run_now', task.run_at < now) - data['scheduled'].append(task) + mapped = add_to_task(task) + if 'error' == mapped: + data['errors'].append(task) + elif mapped: + data['scheduled'].append(task) + + order = getattr(settings, + 'BACKGROUND_TASK_PRIORITY_ORDERING', + 'DESC' + ) + sort_keys = ( + # key, reverse + ('run_now', True), + ('priority', 'ASC' != order), + ('run_at', False), + ) + data['errors'] = multi_key_sort(data['errors'], sort_keys, attr=True) + data['scheduled'] = multi_key_sort(data['scheduled'], sort_keys, attr=True) return data