From 738fca120082bc118c83ecd260f94d3af7011d24 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:01:53 -0400 Subject: [PATCH 01/26] Relaunch the network worker after 12 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run index a9c17d49..60dec45a 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue network + --queue network --duration 43200 From 784acae83ade0afc77de568c509b7110d40fdf3d Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:10:03 -0400 Subject: [PATCH 02/26] Relaunch the database worker after 24 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run index 03b75ea8..f0123c11 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue database + --queue database --duration 86400 --sleep 30 From 7b081a7aafc5b307e7d392c5e3a2c8a4b2ccc0d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:14:42 -0400 Subject: [PATCH 03/26] Relaunch the filesystem worker after 12 hours --- config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run index 0642054d..c5b3cc85 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run @@ -2,4 +2,4 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue filesystem + --queue filesystem --duration 43200 --sleep 20 From d77ca0f7cc2f4c49db42963b3df00dce04256c90 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:19:38 -0400 Subject: [PATCH 04/26] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run index 60dec45a..7f7bcd26 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue network --duration 43200 + --queue network --duration 43200 \ + --sleep "10.${RANDOM}" From 0011a4ef559e00c340995c51842b8f71302f862b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:20:59 -0400 Subject: [PATCH 05/26] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run index c5b3cc85..c0a9fb79 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue filesystem --duration 43200 --sleep 20 + --queue filesystem --duration 43200 \ + --sleep "20.${RANDOM}" From 045bfcbfd629cd4483384f0cf9ff0eb06db0d33e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:22:09 -0400 Subject: [PATCH 06/26] Add a random offset to the sleep value --- config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run index f0123c11..9fbcbc95 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run @@ -2,4 +2,5 @@ exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks \ - --queue database --duration 86400 --sleep 30 + --queue database --duration 86400 \ + --sleep "30.${RANDOM}" From e12aa28a2b63268c00552ec1d6fbbffabe518946 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:33:14 -0400 Subject: [PATCH 07/26] Adjusting `BACKGROUND_TASK_ASYNC_THREADS` is no longer needed --- tubesync/tubesync/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 0ac2b462..c73dbd79 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -212,9 +212,6 @@ if MAX_RUN_TIME < 600: DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 50) -if RENAME_SOURCES or RENAME_ALL_SOURCES: - BACKGROUND_TASK_ASYNC_THREADS += 1 - if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS From 48bca2b996f2db239dc1ef3495be7093a3599591 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 14:45:44 -0400 Subject: [PATCH 08/26] Enable the `ThreadPool` when `TUBESYNC_WORKERS` is used --- tubesync/tubesync/local_settings.py.container | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index cc20f73b..4f386b66 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -62,6 +62,8 @@ else: DEFAULT_THREADS = 1 BACKGROUND_TASK_ASYNC_THREADS = getenv('TUBESYNC_WORKERS', DEFAULT_THREADS, integer=True) +if BACKGROUND_TASK_ASYNC_THREADS > 1: + BACKGROUND_TASK_RUN_ASYNC = True MEDIA_ROOT = CONFIG_BASE_DIR / 'media' From 30633b1aba9c3280242269d6e9b6d1b9002f14be Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 8 Apr 2025 16:10:07 -0400 Subject: [PATCH 09/26] Update README.md --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 17367a4a..71d37863 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has audio stream (such as music) will download to the `audio` directory. All media with a video stream will be downloaded to the `video` directory. All administration of TubeSync is performed via a web interface. You can optionally add a media server, -currently just Plex, to complete the PVR experience. +currently only Jellyfin or Plex, to complete the PVR experience. # Installation @@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab. ### 3. Media Server updating -Currently TubeSync supports Plex as a media server. You can add your local Plex server +Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server under the "media servers" tab. @@ -234,6 +234,13 @@ view these with: $ docker logs --follow tubesync ``` +To include logs with an issue report, please exteact a file and attach it to the issue. +The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container: + +```bash +docker logs -t tubesync > TubeSync.logs.txt 2>&1 +``` + # Advanced usage guides @@ -371,22 +378,26 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_RENAME_SOURCES | Rename media files from selected sources | Source1_directory,Source2_directory | +| TUBESYNC_RENAME_ALL_SOURCES | Rename media files from all sources | True | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| TUBESYNC_SHRINK_NEW | Filter unneeded information from newly retrieved metadata | True | +| TUBESYNC_SHRINK_OLD | Filter unneeded information from metadata loaded from the database | True | +| TUBESYNC_WORKERS | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2 | +| GUNICORN_WORKERS | Number of `gunicorn` (web request) workers to spawn | 3 | +| LISTEN_HOST | IP address for `gunicorn` to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for `gunicorn` to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | postgresql://user:pass@host:port/database | # Manual, non-containerised, installation @@ -396,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.8) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From 8bcb5cafdbb52173c78cabaf566ef633d0d019c7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:12:41 -0400 Subject: [PATCH 10/26] Remove unnecessary kwargs --- tubesync/sync/tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 79e283c3..f3daf876 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -208,9 +208,7 @@ def schedule_media_servers_update(): for mediaserver in MediaServer.objects.all(): rescan_media_server( str(mediaserver.pk), - priority=10, verbose_name=verbose_name.format(mediaserver), - remove_existing_tasks=True, ) @@ -320,7 +318,6 @@ def index_source_task(source_id): verbose_name = _('Downloading metadata for "{}"') download_media_metadata( str(media.pk), - priority=20, verbose_name=verbose_name.format(media.pk), ) # Reset task.verbose_name to the saved value From a62b64c3eee61fdf75462cd289f20371d8bd7945 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:42:30 -0400 Subject: [PATCH 11/26] Adjust FS tasks Move thumbnail download from NET to FS. It does use the network, but not with `yt_dlp` and not in a way that YouTube cares about. --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index f3daf876..0358a6ce 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -479,7 +479,7 @@ def download_media_metadata(media_id): f'{source} / {media}: {media_id}') -@background(schedule=dict(priority=15, run_at=10), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=10, run_at=10), queue=Val(TaskQueue.FS), remove_existing_tasks=True) def download_media_thumbnail(media_id, url): ''' Downloads an image from a URL and save it as a local thumbnail attached to a @@ -654,7 +654,7 @@ def rescan_media_server(mediaserver_id): mediaserver.update() -@background(schedule=dict(priority=25, run_at=600), queue=Val(TaskQueue.FS), remove_existing_tasks=True) +@background(schedule=dict(priority=30, run_at=600), queue=Val(TaskQueue.FS), remove_existing_tasks=True) def save_all_media_for_source(source_id): ''' Iterates all media items linked to a source and saves them to From 3b0ecf29be64addbf3f3932bfb3151c2dd6d3610 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 02:51:57 -0400 Subject: [PATCH 12/26] Move `refesh_formats` to last position in the NET queue --- tubesync/sync/tasks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0358a6ce..95fe5aba 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -240,7 +240,7 @@ def cleanup_removed_media(source, videos): schedule_media_servers_update() -@background(schedule=dict(priority=10, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=20, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -345,7 +345,7 @@ def check_source_directory_exists(source_id): source.make_directory() -@background(schedule=dict(priority=5, run_at=10), queue=Val(TaskQueue.NET)) +@background(schedule=dict(priority=10, run_at=10), queue=Val(TaskQueue.NET)) def download_source_images(source_id): ''' Downloads an image and save it as a local thumbnail attached to a @@ -395,7 +395,7 @@ def download_source_images(source_id): log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}') -@background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=40, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def download_media_metadata(media_id): ''' Downloads the metadata for a media item. @@ -517,7 +517,7 @@ def download_media_thumbnail(media_id, url): return True -@background(schedule=dict(priority=15, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=30, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def download_media(media_id): ''' Downloads the media to disk and attaches it to the Media instance. @@ -707,7 +707,7 @@ def save_all_media_for_source(source_id): update_task_status(task, None) -@background(schedule=dict(priority=10, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) +@background(schedule=dict(priority=50, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) def refesh_formats(media_id): try: media = Media.objects.get(pk=media_id) From 5b53621360dd17a18808d030918695e086226492 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:28:53 -0400 Subject: [PATCH 13/26] Use the `refesh_formats` task instead of the function --- tubesync/sync/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 95fe5aba..292c244f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -634,7 +634,10 @@ def download_media(media_id): log.error(err) # Try refreshing formats if media.has_metadata: - media.refresh_formats + refesh_formats( + str(media.pk), + verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', + ) # Raising an error here triggers the task to be re-attempted (or fail) raise DownloadFailedException(err) From 8816ba3af04eff939464328653b06544c4ab9eb7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:41:13 -0400 Subject: [PATCH 14/26] refesh_formats => refresh_formats --- tubesync/sync/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 292c244f..3fa4f804 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -54,7 +54,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, - 'sync.tasks.refesh_formats': Media, + 'sync.tasks.refresh_formats': Media, 'sync.tasks.rename_media': Media, 'sync.tasks.rename_all_media_for_source': Source, 'sync.tasks.wait_for_media_premiere': Media, @@ -692,7 +692,7 @@ def save_all_media_for_source(source_id): tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' for mn, media in enumerate(refresh_qs, start=1): update_task_status(task, tvn_format.format(mn)) - refesh_formats( + refresh_formats( str(media.pk), verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', ) @@ -711,7 +711,7 @@ def save_all_media_for_source(source_id): @background(schedule=dict(priority=50, run_at=0), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def refesh_formats(media_id): +def refresh_formats(media_id): try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist as e: From d5e588133414e24972610635bd2e100be2efd20a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 9 Apr 2025 03:45:44 -0400 Subject: [PATCH 15/26] fixup: missed one `refesh_formats` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3fa4f804..6b9d7a6b 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -634,7 +634,7 @@ def download_media(media_id): log.error(err) # Try refreshing formats if media.has_metadata: - refesh_formats( + refresh_formats( str(media.pk), verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"', ) From 117e7c2eef7f83da3e0b2a90c73b1f0de9d37a78 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 00:58:45 -0400 Subject: [PATCH 16/26] `3.9` is a better minimum Python version The parser change and a few other things mean that compatibility with `3.8` is unlikely to last much longer. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71d37863..5fc96c4b 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.8) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From ba63cece20bbbdd2582c529540ec0ed269b9941b Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 11 Apr 2025 01:08:00 -0400 Subject: [PATCH 17/26] Revert changes to `README.md` --- README.md | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 5fc96c4b..17367a4a 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has audio stream (such as music) will download to the `audio` directory. All media with a video stream will be downloaded to the `video` directory. All administration of TubeSync is performed via a web interface. You can optionally add a media server, -currently only Jellyfin or Plex, to complete the PVR experience. +currently just Plex, to complete the PVR experience. # Installation @@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab. ### 3. Media Server updating -Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server +Currently TubeSync supports Plex as a media server. You can add your local Plex server under the "media servers" tab. @@ -234,13 +234,6 @@ view these with: $ docker logs --follow tubesync ``` -To include logs with an issue report, please exteact a file and attach it to the issue. -The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container: - -```bash -docker logs -t tubesync > TubeSync.logs.txt 2>&1 -``` - # Advanced usage guides @@ -378,26 +371,22 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------| -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | -| TUBESYNC_RENAME_SOURCES | Rename media files from selected sources | Source1_directory,Source2_directory | -| TUBESYNC_RENAME_ALL_SOURCES | Rename media files from all sources | True | -| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | -| TUBESYNC_SHRINK_NEW | Filter unneeded information from newly retrieved metadata | True | -| TUBESYNC_SHRINK_OLD | Filter unneeded information from metadata loaded from the database | True | -| TUBESYNC_WORKERS | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2 | -| GUNICORN_WORKERS | Number of `gunicorn` (web request) workers to spawn | 3 | -| LISTEN_HOST | IP address for `gunicorn` to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for `gunicorn` to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | postgresql://user:pass@host:port/database | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | # Manual, non-containerised, installation @@ -407,7 +396,7 @@ following this rough guide, you are on your own and should be knowledgeable abou installing and running WSGI-based Python web applications before attempting this. 1. Clone or download this repo -2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv +2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv installed 3. Set up the environment with `pipenv install` 4. Copy `tubesync/tubesync/local_settings.py.example` to From ef1dd0eba8ab61355273fe5ed4ab85d3a24135db Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 09:22:25 -0400 Subject: [PATCH 18/26] Reduce the memory usage --- tubesync/sync/tasks.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 49a5a36f..0d9705bb 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -674,7 +674,10 @@ def save_all_media_for_source(source_id): raise InvalidTaskError(_('no such source')) from e saved_later = set() - mqs = Media.objects.filter(source=source) + mqs = Media.objects.all().defer( + 'metadata', + 'thumb', + ).filter(source=source) task = get_source_check_task(source_id) refresh_qs = mqs.filter( can_download=False, @@ -701,11 +704,17 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media in enumerate(mqs, start=1): - if media.uuid not in saved_later: + for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True), start=1): + if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) - with atomic(): - media.save() + try: + media = Media.objects.get(pk=str(media_uuid)) + except Media.DoesNotExist as e: + log.exception(str(e)) + pass + else: + with atomic(): + media.save() # Reset task.verbose_name to the saved value update_task_status(task, None) From d4c360eaeeeb91fc81aa708fac668b1a15009297 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 10:35:50 -0400 Subject: [PATCH 19/26] Use Django query set iterator function --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0d9705bb..76461daa 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -693,7 +693,7 @@ def save_all_media_for_source(source_id): end=task.verbose_name.find('Check'), ) tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' - for mn, media in enumerate(refresh_qs, start=1): + for mn, media in enumerate(refresh_qs.iterator(chunk_size=1000), start=1): update_task_status(task, tvn_format.format(mn)) refresh_formats( str(media.pk), @@ -704,7 +704,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True), start=1): + for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True).iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: From 7c8c5e2b41f40ec5cba0f8c171af820c732ab10c Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 12 Apr 2025 12:16:54 -0400 Subject: [PATCH 20/26] Use Django query set iterator function some more --- tubesync/sync/tasks.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 76461daa..2252679d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -205,7 +205,7 @@ def schedule_media_servers_update(): # Schedule a task to update media servers log.info(f'Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all(): + for mediaserver in MediaServer.objects.all().iterator(): rescan_media_server( str(mediaserver.pk), verbose_name=verbose_name.format(mediaserver), @@ -214,9 +214,15 @@ def schedule_media_servers_update(): def cleanup_old_media(): with atomic(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): + for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0).iterator(chunk_size=1000): delta = timezone.now() - timedelta(days=source.days_to_keep) - for media in source.media_source.filter(downloaded=True, download_date__lt=delta): + mqs = source.media_source.defer( + 'metadata', + ).filter( + downloaded=True, + download_date__lt=delta, + ) + for media in mqs.iterator(chunk_size=1000): log.info(f'Deleting expired media: {source} / {media} ' f'(now older than {source.days_to_keep} days / ' f'download_date before {delta})') @@ -230,8 +236,12 @@ def cleanup_removed_media(source, videos): if not source.delete_removed_media: return log.info(f'Cleaning up media no longer in source: {source}') - media_objects = Media.objects.filter(source=source) - for media in media_objects: + mqs = Media.objects.defer( + 'metadata', + ).filter( + source=source, + ) + for media in mqs.iterator(chunk_size=1000): matching_source_item = [video['id'] for video in videos if video['id'] == media.key] if not matching_source_item: log.info(f'{media.name} is no longer in source, removing') @@ -766,13 +776,12 @@ def rename_all_media_for_source(source_id): if not create_rename_tasks: return mqs = Media.objects.all().defer( - 'metadata', 'thumb', ).filter( source=source, downloaded=True, ) - for media in mqs: + for media in mqs.iterator(chunk_size=1000): with atomic(): media.rename_files() @@ -816,7 +825,7 @@ def delete_all_media_for_source(source_id, source_name): ).filter( source=source or source_id, ) - for media in mqs: + for media in mqs.iterator(chunk_size=1000): log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): media.delete() From 71c4d63043f749074d864b494d32d1fbf7430da3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 03:53:07 -0400 Subject: [PATCH 21/26] Iterate over independent query sets --- tubesync/sync/tasks.py | 43 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2252679d..92a95e79 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -121,8 +121,7 @@ def update_task_status(task, status): else: task.verbose_name = f'[{status}] {task._verbose_name}' try: - with atomic(): - task.save(update_fields={'verbose_name'}) + task.save(update_fields={'verbose_name'}) except DatabaseError as e: if 'Save with update_fields did not affect any rows.' == str(e): pass @@ -200,16 +199,16 @@ def migrate_queues(): return qs.update(queue=Val(TaskQueue.NET)) +@atomic(durable=False) def schedule_media_servers_update(): - with atomic(): - # Schedule a task to update media servers - log.info(f'Scheduling media server updates') - verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all().iterator(): - rescan_media_server( - str(mediaserver.pk), - verbose_name=verbose_name.format(mediaserver), - ) + # Schedule a task to update media servers + log.info(f'Scheduling media server updates') + verbose_name = _('Request media server rescan for "{}"') + for mediaserver in MediaServer.objects.all().iterator(): + rescan_media_server( + str(mediaserver.pk), + verbose_name=verbose_name.format(mediaserver), + ) def cleanup_old_media(): @@ -684,18 +683,26 @@ def save_all_media_for_source(source_id): raise InvalidTaskError(_('no such source')) from e saved_later = set() - mqs = Media.objects.all().defer( - 'metadata', - 'thumb', - ).filter(source=source) - task = get_source_check_task(source_id) - refresh_qs = mqs.filter( + refresh_qs = Media.objects.all().only( + 'pk', + 'uuid', + 'key', + 'name', + ).filter( + source=source, can_download=False, skip=False, manual_skip=False, downloaded=False, metadata__isnull=False, ) + uuid_qs = Media.objects.all().only( + 'pk', + 'uuid', + ).filter( + source=source, + ).values_list('uuid', flat=True) + task = get_source_check_task(source_id) if task: task._verbose_name = remove_enclosed( task.verbose_name, '[', ']', ' ', @@ -714,7 +721,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{mqs.count():,}' - for mn, media_uuid in enumerate(mqs.values_list('uuid', flat=True).iterator(chunk_size=1000), start=1): + for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: From fa9446c40e1393118d194854f2e6f1bc7429c88e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 04:32:50 -0400 Subject: [PATCH 22/26] `name` property uses `title` column --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 92a95e79..873c2876 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -687,7 +687,7 @@ def save_all_media_for_source(source_id): 'pk', 'uuid', 'key', - 'name', + 'title', # for name property ).filter( source=source, can_download=False, From 3350c0c2e28e85f8bec0630ca22562d4c3a3a148 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 04:43:10 -0400 Subject: [PATCH 23/26] fixup: missed `mqs` => `uuid_qs` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 873c2876..995aab7f 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -720,7 +720,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated - tvn_format = '2/{:,}' + f'/{mqs.count():,}' + tvn_format = '2/{:,}' + f'/{uuid_qs.count():,}' for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) From bb42691d623fb13b43cc399e0ca1f16a53a5aa8d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 05:05:44 -0400 Subject: [PATCH 24/26] Remove all use of the query set `iterator` function The `sqlite` driver appears to be fundamentally incompatible with the Django query set `iterator` function. It only works for the first chunk, then it fails any further database operations because the database is already locked. --- tubesync/sync/tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 995aab7f..20756b7d 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -204,7 +204,7 @@ def schedule_media_servers_update(): # Schedule a task to update media servers log.info(f'Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') - for mediaserver in MediaServer.objects.all().iterator(): + for mediaserver in MediaServer.objects.all(): rescan_media_server( str(mediaserver.pk), verbose_name=verbose_name.format(mediaserver), @@ -213,7 +213,7 @@ def schedule_media_servers_update(): def cleanup_old_media(): with atomic(): - for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0).iterator(chunk_size=1000): + for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0): delta = timezone.now() - timedelta(days=source.days_to_keep) mqs = source.media_source.defer( 'metadata', @@ -221,7 +221,7 @@ def cleanup_old_media(): downloaded=True, download_date__lt=delta, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: log.info(f'Deleting expired media: {source} / {media} ' f'(now older than {source.days_to_keep} days / ' f'download_date before {delta})') @@ -240,7 +240,7 @@ def cleanup_removed_media(source, videos): ).filter( source=source, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: matching_source_item = [video['id'] for video in videos if video['id'] == media.key] if not matching_source_item: log.info(f'{media.name} is no longer in source, removing') @@ -710,7 +710,7 @@ def save_all_media_for_source(source_id): end=task.verbose_name.find('Check'), ) tvn_format = '1/{:,}' + f'/{refresh_qs.count():,}' - for mn, media in enumerate(refresh_qs.iterator(chunk_size=1000), start=1): + for mn, media in enumerate(refresh_qs, start=1): update_task_status(task, tvn_format.format(mn)) refresh_formats( str(media.pk), @@ -721,7 +721,7 @@ def save_all_media_for_source(source_id): # Trigger the post_save signal for each media item linked to this source as various # flags may need to be recalculated tvn_format = '2/{:,}' + f'/{uuid_qs.count():,}' - for mn, media_uuid in enumerate(uuid_qs.iterator(chunk_size=1000), start=1): + for mn, media_uuid in enumerate(uuid_qs, start=1): if media_uuid not in saved_later: update_task_status(task, tvn_format.format(mn)) try: @@ -788,7 +788,7 @@ def rename_all_media_for_source(source_id): source=source, downloaded=True, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: with atomic(): media.rename_files() @@ -832,7 +832,7 @@ def delete_all_media_for_source(source_id, source_name): ).filter( source=source or source_id, ) - for media in mqs.iterator(chunk_size=1000): + for media in mqs: log.info(f'Deleting media for source: {source_name} item: {media.name}') with atomic(): media.delete() From 8e3bfdd3c4a75eac4ea91f268e066f20dea2dc32 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 05:39:01 -0400 Subject: [PATCH 25/26] Don't sleep as long when not using the `ThreadPool` --- tubesync/sync/youtube.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 145e4c5d..61f9a489 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -205,10 +205,14 @@ def get_media_info(url, /, *, days=None, info_json=None): 'paths': paths, 'postprocessors': postprocessors, 'skip_unavailable_fragments': False, - 'sleep_interval_requests': 2 * settings.BACKGROUND_TASK_ASYNC_THREADS, + 'sleep_interval_requests': 1, 'verbose': True if settings.DEBUG else False, 'writeinfojson': True, }) + if settings.BACKGROUND_TASK_RUN_ASYNC: + opts.update({ + 'sleep_interval_requests': 2 * settings.BACKGROUND_TASK_ASYNC_THREADS, + }) if start: log.debug(f'get_media_info: used date range: {opts["daterange"]} for URL: {url}') response = {} From 8a94efda57f2205e00f2a3fbfa9ec21c4d5de545 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 13 Apr 2025 07:47:52 -0400 Subject: [PATCH 26/26] Use `django.db.reset_queries()` in workers --- tubesync/sync/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 20756b7d..f9d0f65c 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -16,7 +16,7 @@ from PIL import Image from django.conf import settings from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import DatabaseError, IntegrityError +from django.db import reset_queries, DatabaseError, IntegrityError from django.db.transaction import atomic from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -254,6 +254,7 @@ def index_source_task(source_id): ''' Indexes media available from a Source object. ''' + reset_queries() cleanup_completed_tasks() # deleting expired media should happen any time an index task is requested cleanup_old_media() @@ -674,6 +675,7 @@ def save_all_media_for_source(source_id): source has its parameters changed and all media needs to be checked to see if its download status has changed. ''' + reset_queries() try: source = Source.objects.get(pk=source_id) except Source.DoesNotExist as e: