Merge pull request #802 from tcely/patch-12

Display task errors from the current page

tubesync/sync/templates/sync/tasks.html

@@ -35,7 +35,7 @@
 </div>
 <div class="row">
   <div class="col s12">
-    <h2>{{ errors|length|intcomma }} Error{{ errors|length|pluralize }}</h2>
+    <h2>{{ total_errors|intcomma }} Total Error{{ total_errors|pluralize }} ({{ errors|length|intcomma }} on this page)</h2>
     <p>
       Tasks which generated an error are shown here. Tasks are retried a couple of
       times, so if there was an intermittent error such as a download got interrupted
@@ -49,14 +49,14 @@
           <i class="fas fa-history"></i> Task will be retried at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
         </a>
       {% empty %}
-        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no tasks with errors.</span>
+        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no tasks with errors on this page.</span>
       {% endfor %}
     </div>
   </div>
 </div>
 <div class="row">
   <div class="col s12">
-    <h2>{{ total_scheduled|intcomma }} Scheduled</h2>
+    <h2>{{ total_scheduled|intcomma }} Scheduled ({{ scheduled|length|intcomma }} on this page)</h2>
     <p>
       Tasks which are scheduled to run in the future or are waiting in a queue to be
       processed. They can be waiting for an available worker to run immediately, or
@@ -70,7 +70,7 @@
           <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}
-        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no scheduled tasks.</span>
+        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no scheduled tasks on this page.</span>
       {% endfor %}
     </div>
   </div>

tubesync/sync/utils.py

@@ -2,7 +2,7 @@ import os
 import re
 import math
 from copy import deepcopy
-from operator import itemgetter
+from operator import attrgetter, itemgetter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 import requests
@@ -179,10 +179,16 @@ def seconds_to_timestr(seconds):
    return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
 
 
-def multi_key_sort(sort_dict, specs, use_reversed=False):
+def multi_key_sort(iterable, specs, /, use_reversed=False, *, item=False, attr=False, key_func=None):
-    result = list(sort_dict)
+    result = list(iterable)
+    if key_func is None:
+        # itemgetter is the default
+        if item or not (item or attr):
+            key_func = itemgetter
+        elif attr:
+            key_func = attrgetter
     for key, reverse in reversed(specs):
-        result = sorted(result, key=itemgetter(key), reverse=reverse)
+        result.sort(key=key_func(key), reverse=reverse)
     if use_reversed:
         return list(reversed(result))
     return result

tubesync/sync/views.py

@@ -27,7 +27,7 @@ from .models import Source, Media, MediaServer
 from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
                     SkipMediaForm, EnableMediaForm, ResetTasksForm,
                     ConfirmDeleteMediaServerForm)
-from .utils import validate_url, delete_file
+from .utils import validate_url, delete_file, multi_key_sort
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
                     delete_task_by_media, index_source_task)
@@ -782,24 +782,41 @@ class TasksView(ListView):
         prefix = '-' if 'ASC' != order else ''
         _priority = f'{prefix}priority'
         return qs.order_by(
-            'run_at',
             _priority,
+            'run_at',
         )
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
         now = timezone.now()
         qs = Task.objects.all()
+        errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True)
+        running_qs = qs.filter(locked_by__isnull=False)
+        scheduled_qs = qs.filter(locked_by__isnull=True)
 
         # Add to context data from ListView
         data['message'] = self.message
         data['source'] = self.filter_source
-        data['running'] = []
+        data['running'] = list()
-        data['errors'] = []
+        data['errors'] = list()
-        data['scheduled'] = []
+        data['total_errors'] = errors_qs.count()
-        data['total_scheduled'] = qs.filter(locked_at__isnull=True).count()
+        data['scheduled'] = list()
+        data['total_scheduled'] = scheduled_qs.count()
 
-        for task in qs.filter(locked_at__isnull=False):
+        def add_to_task(task):
+            obj, url = map_task_to_instance(task)
+            if not obj:
+                return False
+            setattr(task, 'instance', obj)
+            setattr(task, 'url', url)
+            setattr(task, 'run_now', task.run_at < now)
+            if task.has_error():
+                error_message = get_error_message(task)
+                setattr(task, 'error_message', error_message)
+                return 'error'
+            return True
+                    
+        for task in running_qs:
             # There was broken logic in `Task.objects.locked()`, work around it.
             # With that broken logic, the tasks never resume properly.
             # This check unlocks the tasks without a running process.
@@ -807,31 +824,27 @@ class TasksView(ListView):
             # - `True`: locked and PID exists
             # - `False`: locked and PID does not exist
             # - `None`: not `locked_by`, so there was no PID to check
-            if task.locked_by_pid_running() is False:
+            locked_by_pid_running = task.locked_by_pid_running()
+            if locked_by_pid_running is False:
                 task.locked_by = None
                 # do not wait for the task to expire
                 task.locked_at = None
                 task.save()
-            obj, url = map_task_to_instance(task)
+            if locked_by_pid_running and add_to_task(task):
-            if not obj:
-                # Orphaned task, ignore it (it will be deleted when it fires)
-                continue
-            setattr(task, 'instance', obj)
-            setattr(task, 'url', url)
-            setattr(task, 'run_now', task.run_at < now)
-            if task.locked_by_pid_running():
                 data['running'].append(task)
-            elif task.has_error():
+
-                error_message = get_error_message(task)
+        # show all the errors when they fit on one page
-                setattr(task, 'error_message', error_message)
+        if (data['total_errors'] + len(data['running'])) < self.paginate_by:
-                data['errors'].append(task)
+            for task in errors_qs:
-            else:
+                if task in data['running']:
-                data['scheduled'].append(task)
+                    continue
+                mapped = add_to_task(task)
+                if 'error' == mapped:
+                    data['errors'].append(task)
+                elif mapped:
+                    data['scheduled'].append(task)
 
         for task in data['tasks']:
-            obj, url = map_task_to_instance(task)
-            if not obj:
-                continue
             already_added = (
                 task in data['running'] or
                 task in data['errors'] or
@@ -839,10 +852,24 @@ class TasksView(ListView):
             )
             if already_added:
                 continue
-            setattr(task, 'instance', obj)
+            mapped = add_to_task(task)
-            setattr(task, 'url', url)
+            if 'error' == mapped:
-            setattr(task, 'run_now', task.run_at < now)
+                data['errors'].append(task)
-            data['scheduled'].append(task)
+            elif mapped:
+                data['scheduled'].append(task)
+
+        order = getattr(settings,
+            'BACKGROUND_TASK_PRIORITY_ORDERING',
+            'DESC'
+        )
+        sort_keys = (
+            # key, reverse
+            ('run_now', True),
+            ('priority', 'ASC' != order),
+            ('run_at', False),
+        )
+        data['errors'] = multi_key_sort(data['errors'], sort_keys, attr=True)
+        data['scheduled'] = multi_key_sort(data['scheduled'], sort_keys, attr=True)
 
         return data