Improved progress reporting (See desc) (#1125)

* Separate `--console-title` and `--no-progress`
* Add option `--progress` to show progress-bar even in quiet mode
* Fix and refactor `minicurses`
* Use `minicurses` for all progress reporting
* Standardize use of terminal sequences and enable color support for windows 10
* Add option `--progress-template` to customize progress-bar and console-title
* Add postprocessor hooks and progress reporting

Closes: #906, #901, #1085, #1170

yt_dlp/downloader/common.py

@@ -7,7 +7,6 @@ import sys
 import time
 import random
 
-from ..compat import compat_os_name
 from ..utils import (
     decodeArgument,
     encodeFilename,
@@ -17,6 +16,7 @@ from ..utils import (
     timeconvert,
 )
 from ..minicurses import (
+    MultilineLogger,
     MultilinePrinter,
     QuietMultilinePrinter,
     BreaklineStatusPrinter
@@ -44,8 +44,6 @@ class FileDownloader(object):
     noresizebuffer:     Do not automatically resize the download buffer.
     continuedl:         Try to continue downloads if possible.
     noprogress:         Do not print the progress bar.
-    logtostderr:        Log messages to stderr instead of stdout.
-    consoletitle:       Display progress in console window's titlebar.
     nopart:             Do not use temporary .part files.
     updatetime:         Use the Last-modified header to set output file timestamps.
     test:               Download only first bytes to test the downloader.
@@ -61,6 +59,7 @@ class FileDownloader(object):
     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
                         useful for bypassing bandwidth throttling imposed by
                         a webserver (experimental)
+    progress_template:  See YoutubeDL.py
 
     Subclasses of this one must re-define the real_download method.
     """
@@ -73,7 +72,7 @@ class FileDownloader(object):
         self.ydl = ydl
         self._progress_hooks = []
         self.params = params
-        self._multiline = None
+        self._prepare_multiline_status()
         self.add_progress_hook(self.report_progress)
 
     @staticmethod
@@ -242,55 +241,46 @@ class FileDownloader(object):
         """Report destination filename."""
         self.to_screen('[download] Destination: ' + filename)
 
-    def _prepare_multiline_status(self, lines):
-        if self.params.get('quiet'):
+    def _prepare_multiline_status(self, lines=1):
+        if self.params.get('noprogress'):
             self._multiline = QuietMultilinePrinter()
-        elif self.params.get('progress_with_newline', False):
+        elif self.ydl.params.get('logger'):
+            self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
+        elif self.params.get('progress_with_newline'):
             self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
-        elif self.params.get('noprogress', False):
-            self._multiline = None
         else:
-            self._multiline = MultilinePrinter(sys.stderr, lines)
+            self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
 
     def _finish_multiline_status(self):
-        if self._multiline is not None:
-            self._multiline.end()
+        self._multiline.end()
 
-    def _report_progress_status(self, msg, is_last_line=False, progress_line=None):
-        fullmsg = '[download] ' + msg
-        if self.params.get('progress_with_newline', False):
-            self.to_screen(fullmsg)
-        elif progress_line is not None and self._multiline is not None:
-            self._multiline.print_at_line(fullmsg, progress_line)
-        else:
-            if compat_os_name == 'nt' or not sys.stderr.isatty():
-                prev_len = getattr(self, '_report_progress_prev_line_length', 0)
-                if prev_len > len(fullmsg):
-                    fullmsg += ' ' * (prev_len - len(fullmsg))
-                self._report_progress_prev_line_length = len(fullmsg)
-                clear_line = '\r'
-            else:
-                clear_line = '\r\x1b[K'
-            self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
-        self.to_console_title('yt-dlp ' + msg)
+    def _report_progress_status(self, s):
+        progress_dict = s.copy()
+        progress_dict.pop('info_dict')
+        progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
+
+        progress_template = self.params.get('progress_template', {})
+        self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
+            progress_template.get('download') or '[download] %(progress._default_template)s',
+            progress_dict), s.get('progress_idx') or 0)
+        self.to_console_title(self.ydl.evaluate_outtmpl(
+            progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
+            progress_dict))
 
     def report_progress(self, s):
         if s['status'] == 'finished':
-            if self.params.get('noprogress', False):
+            if self.params.get('noprogress'):
                 self.to_screen('[download] Download completed')
-            else:
-                msg_template = '100%%'
-                if s.get('total_bytes') is not None:
-                    s['_total_bytes_str'] = format_bytes(s['total_bytes'])
-                    msg_template += ' of %(_total_bytes_str)s'
-                if s.get('elapsed') is not None:
-                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
-                    msg_template += ' in %(_elapsed_str)s'
-                self._report_progress_status(
-                    msg_template % s, is_last_line=True, progress_line=s.get('progress_idx'))
-            return
-
-        if self.params.get('noprogress'):
+            msg_template = '100%%'
+            if s.get('total_bytes') is not None:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                msg_template += ' of %(_total_bytes_str)s'
+            if s.get('elapsed') is not None:
+                s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                msg_template += ' in %(_elapsed_str)s'
+            s['_percent_str'] = self.format_percent(100)
+            s['_default_template'] = msg_template % s
+            self._report_progress_status(s)
             return
 
         if s['status'] != 'downloading':
@@ -332,8 +322,8 @@ class FileDownloader(object):
                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
             else:
                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
-
-        self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx'))
+        s['_default_template'] = msg_template % s
+        self._report_progress_status(s)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
@@ -405,7 +395,9 @@ class FileDownloader(object):
                     '[download] Sleeping %s seconds ...' % (
                         sleep_interval_sub))
                 time.sleep(sleep_interval_sub)
-        return self.real_download(filename, info_dict), True
+        ret = self.real_download(filename, info_dict)
+        self._finish_multiline_status()
+        return ret, True
 
     def real_download(self, filename, info_dict):
         """Real download process. Redefine in subclasses."""