From a49c0c1b09b5ad7a225c434b5ffd1d7e0e7e018b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 21 Feb 2025 07:26:53 -0500
Subject: [PATCH 01/49] Allow AV1 as a choice

---
 tubesync/sync/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index f3c051fa..033ef45e 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -237,7 +237,7 @@ class Source(models.Model):
         _('source video codec'),
         max_length=8,
         db_index=True,
-        choices=list(reversed(YouTube_VideoCodec.choices[1:])),
+        choices=list(reversed(YouTube_VideoCodec.choices)),
         default=YouTube_VideoCodec.VP9,
         help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)')
     )

From 379d0ff02f3b9a216eefdeb2a46af7ce4d61e227 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 01:14:15 -0400
Subject: [PATCH 02/49] Add `extractor` so that `id` cannot collide

---
 tubesync/sync/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index e63f1e71..c2c35464 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -179,7 +179,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
     playlist_infojson = 'postprocessor_[%(id)s]_%(n_entries)d_%(playlist_count)d_temp'
     outtmpl = dict(
         default='',
-        infojson='%(id)s.%(ext)s' if paths.get('infojson') else '',
+        infojson='%(extractor)s/%(id)s.%(ext)s' if paths.get('infojson') else '',
         pl_infojson=f'{cache_directory_path}/infojson/playlist/{playlist_infojson}.%(ext)s',
     )
     for k in OUTTMPL_TYPES.keys():

From 7002cb458bd1be6fa4089b07372b58ff1f83ce6a Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 03:47:13 -0400
Subject: [PATCH 03/49] Use `list.append` instead of `list.extend`

---
 tubesync/sync/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index c2c35464..e487fa3b 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -170,11 +170,11 @@ def get_media_info(url, /, *, days=None, info_json=None):
         })
     default_postprocessors = user_set('postprocessors', default_opts.__dict__, list())
     postprocessors = user_set('postprocessors', opts, default_postprocessors)
-    postprocessors.extend((dict(
+    postprocessors.append(dict(
         key='Exec',
         when='playlist',
         exec_cmd="/usr/bin/env bash /app/full_playlist.sh '%(id)s' '%(playlist_count)d'",
-    ),))
+    ))
     cache_directory_path = Path(user_set('cachedir', opts, '/dev/shm'))
     playlist_infojson = 'postprocessor_[%(id)s]_%(n_entries)d_%(playlist_count)d_temp'
     outtmpl = dict(

From 9340fa2741c00811936e4c7c9ccfc85f04163633 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 16:28:51 -0400
Subject: [PATCH 04/49] Add `dive` analysis

---
 .github/workflows/ci.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 15a0bf45..914e84fc 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -118,10 +118,12 @@ jobs:
           DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
         run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}"
       - name: Build and push
+        id: build-push
         timeout-minutes: 60
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
+          load: true
           push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }}
           tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:latest
           cache-from: |
@@ -136,3 +138,12 @@ jobs:
             FFMPEG_DATE=${{ env.FFMPEG_DATE }}
             FFMPEG_VERSION=${{ env.FFMPEG_VERSION }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
+      - name: Analysis with `dive`
+        run: |
+          docker run --rm \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            ghcr.io/wagoodman/dive \
+            'docker://${{ steps.build-push.outputs.imageid }}' \
+            --ci \
+            --highestUserWastedPercent '0.03' \
+            --highestWastedBytes '10M'

From a8e425859eb179ec74158b9a26c6468eadc8dbc3 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 16:30:42 -0400
Subject: [PATCH 05/49] Enable the `info` job

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 914e84fc..b0eba921 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -19,7 +19,7 @@ on:
 
 jobs:
   info:
-    if: ${{ !cancelled() && 'pull_request' != github.event_name }}
+    #if: ${{ !cancelled() && 'pull_request' != github.event_name }}
     runs-on: ubuntu-latest
     outputs:
       ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }}

From 6c9dc2fc03a81d78618cbb1f976ddf70171019ec Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 16:48:45 -0400
Subject: [PATCH 06/49] Build with `dive`

---
 .github/workflows/ci.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b0eba921..52201797 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -123,7 +123,6 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
-          load: true
           push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }}
           tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:latest
           cache-from: |
@@ -143,7 +142,7 @@ jobs:
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             ghcr.io/wagoodman/dive \
-            'docker://${{ steps.build-push.outputs.imageid }}' \
+            build -t ${{ env.IMAGE_NAME }} . \
             --ci \
             --highestUserWastedPercent '0.03' \
             --highestWastedBytes '10M'

From 6b280c0cc369c6421b52ce6b8a0a57a6a357f4a5 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 17:22:57 -0400
Subject: [PATCH 07/49] Use a job for `dive`

---
 .github/workflows/ci.yaml | 42 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 52201797..dbbe47a4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -137,12 +137,52 @@ jobs:
             FFMPEG_DATE=${{ env.FFMPEG_DATE }}
             FFMPEG_VERSION=${{ env.FFMPEG_VERSION }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
+  dive:
+    runs-on: ubuntu-latest
+    needs: ['info', 'test']
+    steps:
+      - name: Set environment variables with jq
+        run: |
+          cat >| .ffmpeg.releases.json <<'EOF'
+          ${{ needs.info.outputs.ffmpeg-releases }}
+          EOF
+          mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            var='FFMPEG_DATE' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
+            jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+
+            ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
+            
+            var='FFMPEG_VERSION' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
+            jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+            unset -v delim jq_arg var ;
+          } >> "${GITHUB_ENV}"
+          cat -v "${GITHUB_ENV}"
+          rm -v -f .ffmpeg.releases.json
+      - uses: actions/checkout@v4
       - name: Analysis with `dive`
         run: |
+          docker buildx build \
+            --build-arg IMAGE_NAME=${{ env.IMAGE_NAME }} \
+            --build-arg FFMPEG_DATE=${{ env.FFMPEG_DATE }} \
+            --build-arg FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} }
+            --build-arg YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} \
+            --cache-from type=gha --load \
+            -t ${{ env.IMAGE_NAME }} .
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             ghcr.io/wagoodman/dive \
-            build -t ${{ env.IMAGE_NAME }} . \
+            '${{ env.IMAGE_NAME }}' \
             --ci \
             --highestUserWastedPercent '0.03' \
             --highestWastedBytes '10M'

From 6113824a87638fe4d854ef3e26960676c509dd3b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 17:26:06 -0400
Subject: [PATCH 08/49] fixup: correct line continuation

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index dbbe47a4..fd1d07ce 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -175,7 +175,7 @@ jobs:
           docker buildx build \
             --build-arg IMAGE_NAME=${{ env.IMAGE_NAME }} \
             --build-arg FFMPEG_DATE=${{ env.FFMPEG_DATE }} \
-            --build-arg FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} }
+            --build-arg FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} \
             --build-arg YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} \
             --cache-from type=gha --load \
             -t ${{ env.IMAGE_NAME }} .

From 084d434a94da60740f7b933da7e3368a34b5f035 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 17:58:50 -0400
Subject: [PATCH 09/49] Setup buildx

---
 .github/workflows/ci.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index fd1d07ce..3f7a5cc7 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -139,7 +139,7 @@ jobs:
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
   dive:
     runs-on: ubuntu-latest
-    needs: ['info', 'test']
+    needs: ['info']
     steps:
       - name: Set environment variables with jq
         run: |
@@ -169,8 +169,9 @@ jobs:
           } >> "${GITHUB_ENV}"
           cat -v "${GITHUB_ENV}"
           rm -v -f .ffmpeg.releases.json
+      - uses: docker/setup-buildx-action@v3
       - uses: actions/checkout@v4
-      - name: Analysis with `dive`
+      - name: Build `${{ env.IMAGE_NAME }}` image
         run: |
           docker buildx build \
             --build-arg IMAGE_NAME=${{ env.IMAGE_NAME }} \
@@ -179,6 +180,8 @@ jobs:
             --build-arg YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} \
             --cache-from type=gha --load \
             -t ${{ env.IMAGE_NAME }} .
+      - name: Analysis with `dive`
+        run: |
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             ghcr.io/wagoodman/dive \

From 1b2f87cafbd2f0a7e61fdc05afab5e41174cb5a6 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 18:58:17 -0400
Subject: [PATCH 10/49] Move `jq` work to the `info` job

---
 .github/workflows/ci.yaml | 98 +++++++++++++++------------------------
 1 file changed, 37 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3f7a5cc7..7521fe4e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -22,7 +22,9 @@ jobs:
     #if: ${{ !cancelled() && 'pull_request' != github.event_name }}
     runs-on: ubuntu-latest
     outputs:
+      ffmpeg-date: ${{ steps.jq.outputs.FFMPEG_DATE }}
       ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }}
+      ffmpeg-version: ${{ steps.jq.outputs.FFMPEG_VERSION }}
       lowercase-github-actor: ${{ steps.github-actor.outputs.lowercase }}
       lowercase-github-repository_owner: ${{ steps.github-repository_owner.outputs.lowercase }}
       ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }}
@@ -45,6 +47,35 @@ jobs:
       - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI
         id: yt-dlp
         uses: ./.github/actions/yt-dlp
+      - name: Set outputs with jq
+        id: jq
+        run: |
+          cat >| .ffmpeg.releases.json <<'EOF'
+          ${{ steps.ffmpeg.outputs.releases }}
+          EOF
+          mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            var='FFMPEG_DATE' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
+            jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+
+            ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
+            
+            var='FFMPEG_VERSION' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
+            jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+            unset -v delim jq_arg var ;
+          } >> "${GITHUB_OUTPUT}"
+          cat -v "${GITHUB_OUTPUT}"
+          rm -v -f .ffmpeg.releases.json
 
   test:
     if: ${{ !cancelled() && ( 'pull_request' != github.event_name || (! github.event.pull_request.draft) ) }}
@@ -78,34 +109,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - name: Set environment variables with jq
-        run: |
-          cat >| .ffmpeg.releases.json <<'EOF'
-          ${{ needs.info.outputs.ffmpeg-releases }}
-          EOF
-          mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
-          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
-          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
-          {
-            var='FFMPEG_DATE' ;
-            delim="$(mk_delim "${var}")" ;
-            open_ml_var "${delim}" "${var}" ;
-            jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
-            jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
-            close_ml_var "${delim}" "${var}" ;
-
-            ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
-            
-            var='FFMPEG_VERSION' ;
-            delim="$(mk_delim "${var}")" ;
-            open_ml_var "${delim}" "${var}" ;
-            jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
-            jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
-            close_ml_var "${delim}" "${var}" ;
-            unset -v delim jq_arg var ;
-          } >> "${GITHUB_ENV}"
-          cat -v "${GITHUB_ENV}"
-          rm -v -f .ffmpeg.releases.json
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -134,49 +137,22 @@ jobs:
             ${{ 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'type=inline' || '' }}
           build-args: |
             IMAGE_NAME=${{ env.IMAGE_NAME }}
-            FFMPEG_DATE=${{ env.FFMPEG_DATE }}
-            FFMPEG_VERSION=${{ env.FFMPEG_VERSION }}
+            FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
+            FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
   dive:
-    runs-on: ubuntu-latest
+    if: ${{ !cancelled() && 'success' == needs.info.result }}
     needs: ['info']
+    runs-on: ubuntu-latest
     steps:
-      - name: Set environment variables with jq
-        run: |
-          cat >| .ffmpeg.releases.json <<'EOF'
-          ${{ needs.info.outputs.ffmpeg-releases }}
-          EOF
-          mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
-          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
-          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
-          {
-            var='FFMPEG_DATE' ;
-            delim="$(mk_delim "${var}")" ;
-            open_ml_var "${delim}" "${var}" ;
-            jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
-            jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
-            close_ml_var "${delim}" "${var}" ;
-
-            ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
-            
-            var='FFMPEG_VERSION' ;
-            delim="$(mk_delim "${var}")" ;
-            open_ml_var "${delim}" "${var}" ;
-            jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
-            jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
-            close_ml_var "${delim}" "${var}" ;
-            unset -v delim jq_arg var ;
-          } >> "${GITHUB_ENV}"
-          cat -v "${GITHUB_ENV}"
-          rm -v -f .ffmpeg.releases.json
       - uses: docker/setup-buildx-action@v3
       - uses: actions/checkout@v4
       - name: Build `${{ env.IMAGE_NAME }}` image
         run: |
           docker buildx build \
             --build-arg IMAGE_NAME=${{ env.IMAGE_NAME }} \
-            --build-arg FFMPEG_DATE=${{ env.FFMPEG_DATE }} \
-            --build-arg FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} \
+            --build-arg FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }} \
+            --build-arg FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }} \
             --build-arg YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} \
             --cache-from type=gha --load \
             -t ${{ env.IMAGE_NAME }} .

From 26ba951529cdc95a0372fcfc10831c2f8627c91b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 19:17:31 -0400
Subject: [PATCH 11/49] Use `dive` before push

---
 .github/workflows/ci.yaml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7521fe4e..f4d24aab 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -120,6 +120,29 @@ jobs:
           DOCKER_USERNAME: ${{ github.actor }}
           DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
         run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}"
+      - name: Build image for `dive`
+        id: build-dive-image
+        uses: docker/build-push-action@v6
+        with:
+          build-args: |
+            IMAGE_NAME=${{ env.IMAGE_NAME }}
+            FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
+            FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
+            YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
+          cache-from: type=gha
+          load: true
+          platforms: linux/amd64
+          push: false
+          tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:dive
+      - name: Analysis with `dive`
+        run: |
+          docker run --rm \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            ghcr.io/wagoodman/dive \
+            'ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:dive' \
+            --ci \
+            --highestUserWastedPercent '0.03' \
+            --highestWastedBytes '10M'
       - name: Build and push
         id: build-push
         timeout-minutes: 60

From ad8231d7b9eff11771411108f44b39f51f1c8b36 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 19:24:40 -0400
Subject: [PATCH 12/49] Remove `dive` job

---
 .github/workflows/ci.yaml | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f4d24aab..fe44fc26 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -138,7 +138,7 @@ jobs:
         run: |
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
-            ghcr.io/wagoodman/dive \
+            'ghcr.io/wagoodman/dive' \
             'ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:dive' \
             --ci \
             --highestUserWastedPercent '0.03' \
@@ -163,28 +163,3 @@ jobs:
             FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
             FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
-  dive:
-    if: ${{ !cancelled() && 'success' == needs.info.result }}
-    needs: ['info']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: docker/setup-buildx-action@v3
-      - uses: actions/checkout@v4
-      - name: Build `${{ env.IMAGE_NAME }}` image
-        run: |
-          docker buildx build \
-            --build-arg IMAGE_NAME=${{ env.IMAGE_NAME }} \
-            --build-arg FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }} \
-            --build-arg FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }} \
-            --build-arg YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} \
-            --cache-from type=gha --load \
-            -t ${{ env.IMAGE_NAME }} .
-      - name: Analysis with `dive`
-        run: |
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            ghcr.io/wagoodman/dive \
-            '${{ env.IMAGE_NAME }}' \
-            --ci \
-            --highestUserWastedPercent '0.03' \
-            --highestWastedBytes '10M'

From 2e634511d79a32983f194be92ac790a9cfd4660d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Tue, 1 Apr 2025 19:28:06 -0400
Subject: [PATCH 13/49] Disable the info job

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index fe44fc26..401ae05e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -19,7 +19,7 @@ on:
 
 jobs:
   info:
-    #if: ${{ !cancelled() && 'pull_request' != github.event_name }}
+    if: ${{ !cancelled() && 'pull_request' != github.event_name }}
     runs-on: ubuntu-latest
     outputs:
       ffmpeg-date: ${{ steps.jq.outputs.FFMPEG_DATE }}

From 8ccd4e68c42e164dce1fbe274b5f6b2839e9dd18 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Wed, 2 Apr 2025 08:04:42 -0400
Subject: [PATCH 14/49] Run the check source directory task before saving
 existing sources

---
 tubesync/sync/signals.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index be848a0a..165a1be7 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -30,6 +30,8 @@ def source_pre_save(sender, instance, **kwargs):
         log.debug(f'source_pre_save signal: no existing source: {sender} - {instance}')
         return
 
+    args = ( str(instance.pk), )
+    check_source_directory_exists.now(*args)
     existing_dirpath = existing_source.directory_path.resolve(strict=True)
     new_dirpath = instance.directory_path.resolve(strict=False)
     if existing_dirpath != new_dirpath:

From f5ad4eda16e40de45bcb92a49e50892b0783ffd2 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 3 Apr 2025 00:31:56 -0400
Subject: [PATCH 15/49] Log more details about what `pipenv` installs

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 15a0bf45..966a6664 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -63,7 +63,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pipenv
-          pipenv install --system --skip-lock
+          PIPENV_VERBOSITY=64 pipenv install --system --skip-lock
       - name: Set up Django environment
         run: |
           cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py

From a13e5942ae40a3885f0bdda90aad13338b5dd9db Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 3 Apr 2025 00:37:26 -0400
Subject: [PATCH 16/49] Adjust the test for Django `5.2`

---
 tubesync/sync/tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 514f75b1..b05c3991 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -138,7 +138,7 @@ class FrontEndTestCase(TestCase):
                     else:
                         # Invalid source tests should reload the page with an error
                         self.assertEqual(response.status_code, 200)
-                        self.assertIn('<ul class="errorlist">',
+                        self.assertIn('<ul class="errorlist"',
                                       response.content.decode())
 
     def test_add_source_prepopulation(self):

From 69d3c4326975d3653c1560f16a6e5dcaead301d7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 3 Apr 2025 01:05:08 -0400
Subject: [PATCH 17/49] Avoid Django `5.2` until it is tested

---
 Pipfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Pipfile b/Pipfile
index bf53b4bf..2976db2e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -7,7 +7,7 @@ verify_ssl = true
 autopep8 = "*"
 
 [packages]
-django = "*"
+django = "<5.2"
 django-sass-processor = {extras = ["management-command"], version = "*"}
 pillow = "*"
 whitenoise = "*"

From eab0ad9d7c0a9f2ba1a4b826cdf29bff06b93fb7 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 3 Apr 2025 01:50:20 -0400
Subject: [PATCH 18/49] Review of tasks.py

---
 tubesync/sync/tasks.py | 147 +++++++++++++++++++++--------------------
 1 file changed, 76 insertions(+), 71 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 6cf0fc2d..8e35f7ac 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -21,6 +21,7 @@ from django.db.transaction import atomic
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from background_task import background
+from background_task.exceptions import InvalidTaskError
 from background_task.models import Task, CompletedTask
 from common.logger import log
 from common.errors import NoMediaException, NoMetadataException, DownloadFailedException
@@ -123,7 +124,8 @@ def update_task_status(task, status):
     except DatabaseError as e:
         if 'Save with update_fields did not affect any rows.' == str(e):
             pass
-        raise
+        else:
+            raise
     return True
 
 
@@ -136,6 +138,7 @@ def get_source_completed_tasks(source_id, only_errors=False):
         q['failed_at__isnull'] = False
     return CompletedTask.objects.filter(**q).order_by('-failed_at')
 
+
 def get_tasks(task_name, id=None, /, instance=None):
     assert not (id is None and instance is None)
     arg = str(id or instance.pk)
@@ -160,6 +163,7 @@ def get_source_check_task(source_id):
 def get_source_index_task(source_id):
     return get_first_task('sync.tasks.index_source_task', source_id)
 
+
 def delete_task_by_source(task_name, source_id):
     now = timezone.now()
     unlocked = Task.objects.unlocked(now)
@@ -191,7 +195,7 @@ def schedule_media_servers_update():
         for mediaserver in MediaServer.objects.all():
             rescan_media_server(
                 str(mediaserver.pk),
-                priority=30,
+                priority=10,
                 verbose_name=verbose_name.format(mediaserver),
                 remove_existing_tasks=True,
             )
@@ -225,7 +229,7 @@ def cleanup_removed_media(source, videos):
     schedule_media_servers_update()
 
 
-@background(schedule=300, remove_existing_tasks=True)
+@background(schedule=dict(run_at=300), remove_existing_tasks=True)
 def index_source_task(source_id):
     '''
         Indexes media available from a Source object.
@@ -235,18 +239,20 @@ def index_source_task(source_id):
     cleanup_old_media()
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the Source has been deleted, delete the task
-        return
+        raise InvalidTaskError(_('no such source')) from e
     # An inactive Source would return an empty list for videos anyway
     if not source.is_active:
         return
     # Reset any errors
+    # TODO: determine if this affects anything
     source.has_failed = False
     source.save()
     # Index the source
     videos = source.index_media()
     if not videos:
+        # TODO: Record this error in source.has_failed ?
         raise NoMediaException(f'Source "{source}" (ID: {source_id}) returned no '
                                f'media to index, is the source key valid? Check the '
                                f'source configuration is correct and that the source '
@@ -310,7 +316,7 @@ def index_source_task(source_id):
     cleanup_removed_media(source, videos)
 
 
-@background(schedule=0)
+@background(schedule=dict(run_at=0))
 def check_source_directory_exists(source_id):
     '''
         Checks the output directory for a source exists and is writable, if it does
@@ -319,17 +325,17 @@ def check_source_directory_exists(source_id):
     '''
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the Source has been deleted, delete the task
-        return
+        raise InvalidTaskError(_('no such source')) from e
     # Check the source output directory exists
     if not source.directory_exists():
-        # Try and create it
+        # Try to create it
         log.info(f'Creating directory: {source.directory_path}')
         source.make_directory()
 
 
-@background(schedule=0)
+@background(schedule=dict(run_at=0))
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a
@@ -337,11 +343,11 @@ def download_source_images(source_id):
     '''
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the source no longer exists, do nothing
         log.error(f'Task download_source_images(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
-        return
+        raise InvalidTaskError(_('no such source')) from e
     avatar, banner = source.get_image_url
     log.info(f'Thumbnail URL for source with ID: {source_id} / {source} '
         f'Avatar: {avatar} '
@@ -379,18 +385,18 @@ def download_source_images(source_id):
     log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}')
 
 
-@background(schedule=60, remove_existing_tasks=True)
+@background(schedule=dict(run_at=60), remove_existing_tasks=True)
 def download_media_metadata(media_id):
     '''
         Downloads the metadata for a media item.
     '''
     try:
         media = Media.objects.get(pk=media_id)
-    except Media.DoesNotExist:
+    except Media.DoesNotExist as e:
         # Task triggered but the media no longer exists, do nothing
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
-        return
+        raise InvalidTaskError(_('no such media')) from e
     if media.manual_skip:
         log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.')
         return
@@ -466,7 +472,7 @@ def download_media_metadata(media_id):
              f'{source} / {media}: {media_id}')
 
 
-@background(schedule=60, remove_existing_tasks=True)
+@background(schedule=dict(run_at=60), remove_existing_tasks=True)
 def download_media_thumbnail(media_id, url):
     '''
         Downloads an image from a URL and save it as a local thumbnail attached to a
@@ -474,10 +480,10 @@ def download_media_thumbnail(media_id, url):
     '''
     try:
         media = Media.objects.get(pk=media_id)
-    except Media.DoesNotExist:
+    except Media.DoesNotExist as e:
         # Task triggered but the media no longer exists, do nothing
-        return
-    if media.skip:
+        raise InvalidTaskError(_('no such media')) from e
+    if media.skip or media.manual_skip:
         # Media was toggled to be skipped after the task was scheduled
         log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
                  f'it is now marked to be skipped, not downloading thumbnail')
@@ -504,38 +510,43 @@ def download_media_thumbnail(media_id, url):
     return True
 
 
-@background(schedule=60, remove_existing_tasks=True)
+@background(schedule=dict(run_at=60), remove_existing_tasks=True)
 def download_media(media_id):
     '''
         Downloads the media to disk and attaches it to the Media instance.
     '''
     try:
         media = Media.objects.get(pk=media_id)
-    except Media.DoesNotExist:
+    except Media.DoesNotExist as e:
         # Task triggered but the media no longer exists, do nothing
-        return
-    if not media.has_metadata:
-        raise NoMetadataException('Metadata is not yet available.')
-    if media.skip:
-        # Media was toggled to be skipped after the task was scheduled
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
-                 f'it is now marked to be skipped, not downloading')
-        return
-    downloaded_file_exists = (
-        media.media_file_exists or
-        media.filepath.exists()
-    )
-    if media.downloaded and downloaded_file_exists:
-        # Media has been marked as downloaded before the download_media task was fired,
-        # skip it
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
-                 f'it has already been marked as downloaded, not downloading again')
-        return
+        raise InvalidTaskError(_('no such media')) from e
     if not media.source.download_media:
         log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
                  f'the source {media.source} has since been marked to not download, '
                  f'not downloading')
         return
+    if media.skip or media.manual_skip:
+        # Media was toggled to be skipped after the task was scheduled
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'it is now marked to be skipped, not downloading')
+        return
+    # metadata is required to generate the proper filepath
+    if not media.has_metadata:
+        raise NoMetadataException('Metadata is not yet available.')
+    downloaded_file_exists = (
+        media.downloaded and
+        media.has_metadata and
+        (
+            media.media_file_exists or
+            media.filepath.exists()
+        )
+    )
+    if downloaded_file_exists:
+        # Media has been marked as downloaded before the download_media task was fired,
+        # skip it
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'it has already been marked as downloaded, not downloading again')
+        return
     max_cap_age = media.source.download_cap_date
     published = media.published
     if max_cap_age and published:
@@ -608,16 +619,7 @@ def download_media(media_id):
                 log.warn(f'A permissions problem occured when writing the new media NFO file: {e.msg}')
                 pass
         # Schedule a task to update media servers
-        for mediaserver in MediaServer.objects.all():
-            log.info(f'Scheduling media server updates')
-            verbose_name = _('Request media server rescan for "{}"')
-            rescan_media_server(
-                str(mediaserver.pk),
-                queue=str(media.source.pk),
-                priority=0,
-                verbose_name=verbose_name.format(mediaserver),
-                remove_existing_tasks=True
-            )
+        schedule_media_servers_update()
     else:
         # Expected file doesn't exist on disk
         err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
@@ -630,22 +632,22 @@ def download_media(media_id):
         raise DownloadFailedException(err)
 
 
-@background(schedule=300, remove_existing_tasks=True)
+@background(schedule=dict(run_at=300), remove_existing_tasks=True)
 def rescan_media_server(mediaserver_id):
     '''
         Attempts to request a media rescan on a remote media server.
     '''
     try:
         mediaserver = MediaServer.objects.get(pk=mediaserver_id)
-    except MediaServer.DoesNotExist:
+    except MediaServer.DoesNotExist as e:
         # Task triggered but the media server no longer exists, do nothing
-        return
+        raise InvalidTaskError(_('no such server')) from e
     # Request an rescan / update
     log.info(f'Updating media server: {mediaserver}')
     mediaserver.update()
 
 
-@background(schedule=300, remove_existing_tasks=True)
+@background(schedule=dict(run_at=300), remove_existing_tasks=True)
 def save_all_media_for_source(source_id):
     '''
         Iterates all media items linked to a source and saves them to
@@ -655,11 +657,11 @@ def save_all_media_for_source(source_id):
     '''
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the source no longer exists, do nothing
         log.error(f'Task save_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
-        return
+        raise InvalidTaskError(_('no such source')) from e
 
     already_saved = set()
     mqs = Media.objects.filter(source=source)
@@ -694,41 +696,41 @@ def save_all_media_for_source(source_id):
     # flags may need to be recalculated
     tvn_format = '2/{:,}' + f'/{mqs.count():,}'
     for mn, media in enumerate(mqs, start=1):
-        update_task_status(task, tvn_format.format(mn))
         if media.uuid not in already_saved:
+            update_task_status(task, tvn_format.format(mn))
             with atomic():
                 media.save()
     # Reset task.verbose_name to the saved value
     update_task_status(task, None)
 
 
-@background(schedule=60, remove_existing_tasks=True)
+@background(schedule=dict(run_at=60), remove_existing_tasks=True)
 def rename_media(media_id):
     try:
         media = Media.objects.defer('metadata', 'thumb').get(pk=media_id)
-    except Media.DoesNotExist:
-        return
+    except Media.DoesNotExist as e:
+        raise InvalidTaskError(_('no such media')) from e
     media.rename_files()
 
 
-@background(schedule=300, remove_existing_tasks=True)
+@background(schedule=dict(run_at=300), remove_existing_tasks=True)
 @atomic(durable=True)
 def rename_all_media_for_source(source_id):
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the source no longer exists, do nothing
         log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
-        return
+        raise InvalidTaskError(_('no such source')) from e
     # Check that the settings allow renaming
-    rename_sources_setting = settings.RENAME_SOURCES or list()
+    rename_sources_setting = getattr(settings, 'RENAME_SOURCES', list())
     create_rename_tasks = (
         (
             source.directory and
             source.directory in rename_sources_setting
         ) or
-        settings.RENAME_ALL_SOURCES
+        getattr(settings, 'RENAME_ALL_SOURCES', False)
     )
     if not create_rename_tasks:
         return
@@ -744,15 +746,15 @@ def rename_all_media_for_source(source_id):
             media.rename_files()
 
 
-@background(schedule=60, remove_existing_tasks=True)
+@background(schedule=dict(run_at=60), remove_existing_tasks=True)
 def wait_for_media_premiere(media_id):
     hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
 
     try:
         media = Media.objects.get(pk=media_id)
-    except Media.DoesNotExist:
-        return
-    if media.metadata:
+    except Media.DoesNotExist as e:
+        raise InvalidTaskError(_('no such media')) from e
+    if media.has_metadata:
         return
     now = timezone.now()
     if media.published < now:
@@ -764,17 +766,20 @@ def wait_for_media_premiere(media_id):
         media.manual_skip = True
         media.title = _(f'Premieres in {hours(media.published - now)} hours')
         media.save()
+        task = get_media_premiere_task(media_id)
+        if task:
+            update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=300, remove_existing_tasks=False)
+@background(schedule=dict(run_at=300), remove_existing_tasks=False)
 def delete_all_media_for_source(source_id, source_name):
     source = None
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist:
+    except Source.DoesNotExist as e:
         # Task triggered but the source no longer exists, do nothing
         log.error(f'Task delete_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
-        pass
+        raise InvalidTaskError(_('no such source')) from e
     mqs = Media.objects.all().defer(
         'metadata',
     ).filter(

From 6293625a019f9fb80cd1d9826f51f1d89e68e322 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Thu, 3 Apr 2025 15:19:08 -0400
Subject: [PATCH 19/49] Clean up old `debconf` cache files

---
 Dockerfile | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 82bc665e..d3169884 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,7 +47,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
     locale-gen en_US.UTF-8 && \
     # Clean up
     apt-get -y autopurge && \
-    apt-get -y autoclean
+    apt-get -y autoclean && \
+    rm -f /var/cache/debconf/*.dat-old
 
 FROM alpine:${ALPINE_VERSION} AS ffmpeg-download
 ARG FFMPEG_DATE
@@ -289,7 +290,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
   useradd -M -d /app -s /bin/false -g app app && \
   # Clean up
   apt-get -y autopurge && \
-  apt-get -y autoclean
+  apt-get -y autoclean && \
+  rm -v -f /var/cache/debconf/*.dat-old
 
 # Install third party software
 COPY --from=s6-overlay / /
@@ -310,7 +312,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
     apt-get -y autoremove --purge file && \
     # Clean up
     apt-get -y autopurge && \
-    apt-get -y autoclean
+    apt-get -y autoclean && \
+    rm -v -f /var/cache/debconf/*.dat-old
 
 # Switch workdir to the the app
 WORKDIR /app
@@ -362,6 +365,7 @@ RUN --mount=type=tmpfs,target=/cache \
   && \
   apt-get -y autopurge && \
   apt-get -y autoclean && \
+  rm -v -f /var/cache/debconf/*.dat-old && \
   rm -v -rf /tmp/*
 
 # Copy root

From ec41580fe9ab69dbbe5958b2550ba840bb01547b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Fri, 4 Apr 2025 16:33:41 -0400
Subject: [PATCH 20/49] Close the `ThreadPool` before exiting

---
 .../management/commands/process_tasks.py      | 124 ++++++++++++++++++
 1 file changed, 124 insertions(+)
 create mode 100644 patches/background_task/management/commands/process_tasks.py

diff --git a/patches/background_task/management/commands/process_tasks.py b/patches/background_task/management/commands/process_tasks.py
new file mode 100644
index 00000000..9484c393
--- /dev/null
+++ b/patches/background_task/management/commands/process_tasks.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+import logging
+import random
+import sys
+import time
+
+from django import VERSION
+from django.core.management.base import BaseCommand
+from django.utils import autoreload
+
+from background_task.tasks import tasks, autodiscover
+from background_task.utils import SignalManager
+from django.db import close_old_connections as close_connection
+
+
+logger = logging.getLogger(__name__)
+
+
+def _configure_log_std():
+    class StdOutWrapper(object):
+        def write(self, s):
+            logger.info(s)
+
+    class StdErrWrapper(object):
+        def write(self, s):
+            logger.error(s)
+    sys.stdout = StdOutWrapper()
+    sys.stderr = StdErrWrapper()
+
+
+class Command(BaseCommand):
+    help = 'Run tasks that are scheduled to run on the queue'
+
+    # Command options are specified in an abstract way to enable Django < 1.8 compatibility
+    OPTIONS = (
+        (('--duration', ), {
+            'action': 'store',
+            'dest': 'duration',
+            'type': int,
+            'default': 0,
+            'help': 'Run task for this many seconds (0 or less to run forever) - default is 0',
+        }),
+        (('--sleep', ), {
+            'action': 'store',
+            'dest': 'sleep',
+            'type': float,
+            'default': 5.0,
+            'help': 'Sleep for this many seconds before checking for new tasks (if none were found) - default is 5',
+        }),
+        (('--queue', ), {
+            'action': 'store',
+            'dest': 'queue',
+            'help': 'Only process tasks on this named queue',
+        }),
+        (('--log-std', ), {
+            'action': 'store_true',
+            'dest': 'log_std',
+            'help': 'Redirect stdout and stderr to the logging system',
+        }),
+        (('--dev', ), {
+            'action': 'store_true',
+            'dest': 'dev',
+            'help': 'Auto-reload your code on changes. Use this only for development',
+        }),
+    )
+
+    if VERSION < (1, 8):
+        from optparse import make_option
+        option_list = BaseCommand.option_list + tuple([make_option(*args, **kwargs) for args, kwargs in OPTIONS])
+
+    # Used in Django >= 1.8
+    def add_arguments(self, parser):
+        for (args, kwargs) in self.OPTIONS:
+            parser.add_argument(*args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+        self.sig_manager = None
+        self._tasks = tasks
+
+    def run(self, *args, **options):
+        duration = options.get('duration', 0)
+        sleep = options.get('sleep', 5.0)
+        queue = options.get('queue', None)
+        log_std = options.get('log_std', False)
+        is_dev = options.get('dev', False)
+        sig_manager = self.sig_manager
+
+        if is_dev:
+            # raise last Exception is exist
+            autoreload.raise_last_exception()
+
+        if log_std:
+            _configure_log_std()
+
+        autodiscover()
+
+        start_time = time.time()
+
+        while (duration <= 0) or (time.time() - start_time) <= duration:
+            if sig_manager.kill_now:
+                # shutting down gracefully
+                break
+
+            if not self._tasks.run_next_task(queue):
+                # there were no tasks in the queue, let's recover.
+                close_connection()
+                logger.debug('waiting for tasks')
+                time.sleep(sleep)
+            else:
+                # there were some tasks to process, let's check if there is more work to do after a little break.
+                time.sleep(random.uniform(sig_manager.time_to_wait[0], sig_manager.time_to_wait[1]))
+        self._tasks._pool_runner._pool.close()
+
+    def handle(self, *args, **options):
+        is_dev = options.get('dev', False)
+        self.sig_manager = SignalManager()
+        if is_dev:
+            reload_func = autoreload.run_with_reloader
+            if VERSION < (2, 2):
+                reload_func = autoreload.main
+            reload_func(self.run, *args, **options)
+        else:
+            self.run(*args, **options)

From ddf985ff7d3d193fa7a9205308e76a68354e29a3 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 13:47:16 -0400
Subject: [PATCH 21/49] Use `schedule_media_servers_update` function

---
 tubesync/sync/management/commands/delete-source.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py
index 206aee7f..5ab8a325 100644
--- a/tubesync/sync/management/commands/delete-source.py
+++ b/tubesync/sync/management/commands/delete-source.py
@@ -6,7 +6,7 @@ from django.db.models import signals
 from common.logger import log
 from sync.models import Source, Media, MediaServer
 from sync.signals import media_post_delete
-from sync.tasks import rescan_media_server
+from sync.tasks import schedule_media_servers_update
 
 
 class Command(BaseCommand):
@@ -37,15 +37,6 @@ class Command(BaseCommand):
         log.info(f'Source directory: {source.directory_path}')
         source.delete()
         # Update any media servers
-        for mediaserver in MediaServer.objects.all():
-            log.info(f'Scheduling media server updates')
-            verbose_name = _('Request media server rescan for "{}"')
-            rescan_media_server(
-                str(mediaserver.pk),
-                priority=0,
-                schedule=30,
-                verbose_name=verbose_name.format(mediaserver),
-                remove_existing_tasks=True
-            )
+        schedule_media_servers_update()
         # All done
         log.info('Done')

From e0bbb5951b25cd5c8fdfead7a0a9423bba7c47af Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 14:28:34 -0400
Subject: [PATCH 22/49] Remove `priority` kwarg

---
 tubesync/sync/management/commands/reset-tasks.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py
index 7d78c09f..3d5ecb98 100644
--- a/tubesync/sync/management/commands/reset-tasks.py
+++ b/tubesync/sync/management/commands/reset-tasks.py
@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand, CommandError
+from django.db.transaction import atomic
 from django.utils.translation import gettext_lazy as _
 from background_task.models import Task
 from sync.models import Source
@@ -12,6 +13,7 @@ class Command(BaseCommand):
 
     help = 'Resets all tasks'
 
+    @atomic(durable=True)
     def handle(self, *args, **options):
         log.info('Resettings all tasks...')
         # Delete all tasks
@@ -23,9 +25,8 @@ class Command(BaseCommand):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(source.pk),
-                repeat=source.index_schedule,
                 queue=str(source.pk),
-                priority=10,
+                repeat=source.index_schedule,
                 verbose_name=verbose_name.format(source.name)
             )
             # This also chains down to call each Media objects .save() as well

From 07390a32a813ae93190598c354d12c4e289aa893 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 14:33:12 -0400
Subject: [PATCH 23/49] Remove `priority` kwarg

---
 tubesync/sync/views.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 3d1896d2..bfdcbf6f 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -123,8 +123,6 @@ class SourcesView(ListView):
                 str(sobj.pk),
                 queue=str(sobj.pk),
                 repeat=0,
-                priority=10,
-                schedule=30,
                 remove_existing_tasks=False,
                 verbose_name=verbose_name.format(sobj.name))
             url = reverse_lazy('sync:sources')
@@ -932,9 +930,8 @@ class ResetTasks(FormView):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(source.pk),
-                repeat=source.index_schedule,
                 queue=str(source.pk),
-                priority=10,
+                repeat=source.index_schedule,
                 verbose_name=verbose_name.format(source.name)
             )
             # This also chains down to call each Media objects .save() as well

From 4c087269062c66f308d9e9cc5b576fc7b288d026 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 14:42:21 -0400
Subject: [PATCH 24/49] priority: 10: index_source_task

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 8e35f7ac..fb7a0892 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -229,7 +229,7 @@ def cleanup_removed_media(source, videos):
     schedule_media_servers_update()
 
 
-@background(schedule=dict(run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=10, run_at=30), remove_existing_tasks=True)
 def index_source_task(source_id):
     '''
         Indexes media available from a Source object.

From 017966160d74304130f4a1de7baef8b4b9c7ba25 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 15:55:46 -0400
Subject: [PATCH 25/49] Remove `priority` kwarg

---
 tubesync/sync/signals.py | 37 ++++++++++++-------------------------
 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 165a1be7..90b39480 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -92,12 +92,10 @@ def source_pre_save(sender, instance, **kwargs):
         verbose_name = _('Index media from source "{}"')
         index_source_task(
             str(instance.pk),
-            schedule=instance.index_schedule,
-            repeat=instance.index_schedule,
             queue=str(instance.pk),
-            priority=10,
+            repeat=instance.index_schedule,
+            schedule=instance.index_schedule,
             verbose_name=verbose_name.format(instance.name),
-            remove_existing_tasks=True
         )
 
 
@@ -108,14 +106,14 @@ def source_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Check download directory exists for source "{}"')
         check_source_directory_exists(
             str(instance.pk),
-            priority=0,
-            verbose_name=verbose_name.format(instance.name)
+            queue=str(instance.pk),
+            verbose_name=verbose_name.format(instance.name),
         )
         if instance.source_type != Val(YouTube_SourceType.PLAYLIST) and instance.copy_channel_images:
             download_source_images(
                 str(instance.pk),
-                priority=5,
-                verbose_name=verbose_name.format(instance.name)
+                queue=str(instance.pk),
+                verbose_name=verbose_name.format(instance.name),
             )
         if instance.index_schedule > 0:
             delete_task_by_source('sync.tasks.index_source_task', instance.pk)
@@ -123,20 +121,17 @@ def source_post_save(sender, instance, created, **kwargs):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(instance.pk),
-                schedule=600,
-                repeat=instance.index_schedule,
                 queue=str(instance.pk),
-                priority=10,
+                repeat=instance.index_schedule,
+                schedule=600,
                 verbose_name=verbose_name.format(instance.name),
-                remove_existing_tasks=True
             )
 
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
-        priority=25,
+        queue=str(instance.pk),
         verbose_name=verbose_name.format(instance.name),
-        remove_existing_tasks=True
     )
 
 
@@ -157,7 +152,6 @@ def source_pre_delete(sender, instance, **kwargs):
     delete_all_media_for_source(
         str(instance.pk),
         str(instance.name),
-        priority=1,
         verbose_name=verbose_name.format(instance.name),
     )
     # Try to do it all immediately
@@ -245,9 +239,7 @@ def media_post_save(sender, instance, created, **kwargs):
             rename_media(
                 str(media.pk),
                 queue=str(media.pk),
-                priority=20,
                 verbose_name=verbose_name.format(media.key, media.name),
-                remove_existing_tasks=True
             )
 
     # If the media is missing metadata schedule it to be downloaded
@@ -256,9 +248,8 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            priority=20,
+            queue=str(media.pk),
             verbose_name=verbose_name.format(instance.pk),
-            remove_existing_tasks=True
         )
     # If the media is missing a thumbnail schedule it to be downloaded (unless we are skipping this media)
     if not instance.thumb_file_exists:
@@ -272,10 +263,8 @@ def media_post_save(sender, instance, created, **kwargs):
             download_media_thumbnail(
                 str(instance.pk),
                 thumbnail_url,
-                queue=str(instance.source.pk),
-                priority=15,
+                queue=str(instance.pk),
                 verbose_name=verbose_name.format(instance.name),
-                remove_existing_tasks=True
             )
     # If the media has not yet been downloaded schedule it to be downloaded
     if not (instance.media_file_exists or instance.filepath.exists() or existing_media_download_task):
@@ -289,10 +278,8 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading media for "{}"')
         download_media(
             str(instance.pk),
-            queue=str(instance.source.pk),
-            priority=15,
+            queue=str(instance.pk),
             verbose_name=verbose_name.format(instance.name),
-            remove_existing_tasks=True
         )
     # Save the instance if any changes were required
     if skip_changed or can_download_changed:

From 82688a8475e9c8fa02feb6fd827cec89137d314d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 16:11:08 -0400
Subject: [PATCH 26/49] Restore `schedule` kwarg

---
 tubesync/sync/views.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index bfdcbf6f..5e937e5e 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -118,13 +118,16 @@ class SourcesView(ListView):
             if sobj is None:
                 return HttpResponseNotFound()
 
+            source = sobj
             verbose_name = _('Index media from source "{}" once')
             index_source_task(
-                str(sobj.pk),
-                queue=str(sobj.pk),
-                repeat=0,
+                str(source.pk),
+                queue=str(source.pk),
                 remove_existing_tasks=False,
-                verbose_name=verbose_name.format(sobj.name))
+                repeat=0,
+                schedule=30,
+                verbose_name=verbose_name.format(source.name),
+            )
             url = reverse_lazy('sync:sources')
             url = append_uri_params(url, {'message': 'source-refreshed'})
             return HttpResponseRedirect(url)

From 52579865b2e9f01d06b9c4cece2d72da31ab0b8d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 16:27:18 -0400
Subject: [PATCH 27/49] priority: 01: delete_all_media_for_source

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index fb7a0892..d335eb31 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -770,7 +770,7 @@ def wait_for_media_premiere(media_id):
         if task:
             update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=dict(run_at=300), remove_existing_tasks=False)
+@background(schedule=dict(priority=1, run_at=300), remove_existing_tasks=False)
 def delete_all_media_for_source(source_id, source_name):
     source = None
     try:

From 61623b66abeb4f7c4de4594578468a70cf8834cb Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 16:33:31 -0400
Subject: [PATCH 28/49] priority: 05: download_source_images

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index d335eb31..7aaa8a8e 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -335,7 +335,7 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
-@background(schedule=dict(run_at=0))
+@background(schedule=dict(priority=5, run_at=0))
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a

From 11baadc6efac3b75e32db9e6dbb66d9e0f39a0ff Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 16:37:09 -0400
Subject: [PATCH 29/49] priority: 25: save_all_media_for_source

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 7aaa8a8e..f7986f70 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -647,7 +647,7 @@ def rescan_media_server(mediaserver_id):
     mediaserver.update()
 
 
-@background(schedule=dict(run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=25, run_at=300), remove_existing_tasks=True)
 def save_all_media_for_source(source_id):
     '''
         Iterates all media items linked to a source and saves them to

From 1af9070d1c0ded0217ab5e1aea9890dcbdeaa2dd Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 16:56:24 -0400
Subject: [PATCH 30/49] priority: 20: rename_media

---
 tubesync/sync/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index f7986f70..897257ba 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -704,7 +704,7 @@ def save_all_media_for_source(source_id):
     update_task_status(task, None)
 
 
-@background(schedule=dict(run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True)
 def rename_media(media_id):
     try:
         media = Media.objects.defer('metadata', 'thumb').get(pk=media_id)
@@ -713,7 +713,7 @@ def rename_media(media_id):
     media.rename_files()
 
 
-@background(schedule=dict(run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=300), remove_existing_tasks=True)
 @atomic(durable=True)
 def rename_all_media_for_source(source_id):
     try:

From c8a9037fb20a20ec15bbfcbf03e788c320f2c494 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 17:15:09 -0400
Subject: [PATCH 31/49] priority: 20: download_media_metadata

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 897257ba..3675421a 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -385,7 +385,7 @@ def download_source_images(source_id):
     log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}')
 
 
-@background(schedule=dict(run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True)
 def download_media_metadata(media_id):
     '''
         Downloads the metadata for a media item.

From d769f39a86be7412a19c962380a208ce909c0a79 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 17:20:27 -0400
Subject: [PATCH 32/49] priority: 15: download_media_thumbnail

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 3675421a..b2e15aee 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -472,7 +472,7 @@ def download_media_metadata(media_id):
              f'{source} / {media}: {media_id}')
 
 
-@background(schedule=dict(run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=60), remove_existing_tasks=True)
 def download_media_thumbnail(media_id, url):
     '''
         Downloads an image from a URL and save it as a local thumbnail attached to a

From 8e3523ae9c7fa437c8f71dd64065a8d742dea64d Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 17:23:56 -0400
Subject: [PATCH 33/49] priority: 15: download_media

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index b2e15aee..a172d098 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -510,7 +510,7 @@ def download_media_thumbnail(media_id, url):
     return True
 
 
-@background(schedule=dict(run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=60), remove_existing_tasks=True)
 def download_media(media_id):
     '''
         Downloads the media to disk and attaches it to the Media instance.

From 529f3cbbd0591b95cf06eadbd58cbef697968088 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 17:40:10 -0400
Subject: [PATCH 34/49] Add priority kwarg for default value

---
 tubesync/sync/tasks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index a172d098..b539a2f9 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -316,7 +316,7 @@ def index_source_task(source_id):
     cleanup_removed_media(source, videos)
 
 
-@background(schedule=dict(run_at=0))
+@background(schedule=dict(priority=0, run_at=0))
 def check_source_directory_exists(source_id):
     '''
         Checks the output directory for a source exists and is writable, if it does
@@ -335,7 +335,7 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
-@background(schedule=dict(priority=5, run_at=0))
+@background(schedule=dict(priority=5, run_at=10))
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a
@@ -472,7 +472,7 @@ def download_media_metadata(media_id):
              f'{source} / {media}: {media_id}')
 
 
-@background(schedule=dict(priority=15, run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=10), remove_existing_tasks=True)
 def download_media_thumbnail(media_id, url):
     '''
         Downloads an image from a URL and save it as a local thumbnail attached to a
@@ -632,7 +632,7 @@ def download_media(media_id):
         raise DownloadFailedException(err)
 
 
-@background(schedule=dict(run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=30), remove_existing_tasks=True)
 def rescan_media_server(mediaserver_id):
     '''
         Attempts to request a media rescan on a remote media server.
@@ -647,7 +647,7 @@ def rescan_media_server(mediaserver_id):
     mediaserver.update()
 
 
-@background(schedule=dict(priority=25, run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=25, run_at=600), remove_existing_tasks=True)
 def save_all_media_for_source(source_id):
     '''
         Iterates all media items linked to a source and saves them to
@@ -746,7 +746,7 @@ def rename_all_media_for_source(source_id):
             media.rename_files()
 
 
-@background(schedule=dict(run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=60), remove_existing_tasks=True)
 def wait_for_media_premiere(media_id):
     hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
 

From 7ac5f2c148c2b8ea184b35b3f14f0d9732712683 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 17:53:58 -0400
Subject: [PATCH 35/49] Update for adjusted queue

---
 tubesync/sync/tests.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index b05c3991..c16c4954 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -420,8 +420,7 @@ class FrontEndTestCase(TestCase):
         found_download_task1 = False
         found_download_task2 = False
         found_download_task3 = False
-        q = {'queue': str(test_source.pk),
-             'task_name': 'sync.tasks.download_media_thumbnail'}
+        q = {'task_name': 'sync.tasks.download_media_thumbnail'}
         for task in Task.objects.filter(**q):
             if test_media1_pk in task.task_params:
                 found_thumbnail_task1 = True
@@ -429,8 +428,7 @@ class FrontEndTestCase(TestCase):
                 found_thumbnail_task2 = True
             if test_media3_pk in task.task_params:
                 found_thumbnail_task3 = True
-        q = {'queue': str(test_source.pk),
-             'task_name': 'sync.tasks.download_media'}
+        q = {'task_name': 'sync.tasks.download_media'}
         for task in Task.objects.filter(**q):
             if test_media1_pk in task.task_params:
                 found_download_task1 = True

From 0b92ae0500aa1675c7ff26f606faa7783d6f8ecc Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 18:42:07 -0400
Subject: [PATCH 36/49] Assign task queues based on resources used

---
 tubesync/sync/tasks.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index b539a2f9..aec69216 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -229,7 +229,7 @@ def cleanup_removed_media(source, videos):
     schedule_media_servers_update()
 
 
-@background(schedule=dict(priority=10, run_at=30), remove_existing_tasks=True)
+@background(schedule=dict(priority=10, run_at=30), queue='network', remove_existing_tasks=True)
 def index_source_task(source_id):
     '''
         Indexes media available from a Source object.
@@ -316,7 +316,7 @@ def index_source_task(source_id):
     cleanup_removed_media(source, videos)
 
 
-@background(schedule=dict(priority=0, run_at=0))
+@background(schedule=dict(priority=0, run_at=0), queue='filesystem')
 def check_source_directory_exists(source_id):
     '''
         Checks the output directory for a source exists and is writable, if it does
@@ -335,7 +335,7 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
-@background(schedule=dict(priority=5, run_at=10))
+@background(schedule=dict(priority=5, run_at=10), queue='network')
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a
@@ -385,7 +385,7 @@ def download_source_images(source_id):
     log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}')
 
 
-@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), queue='network', remove_existing_tasks=True)
 def download_media_metadata(media_id):
     '''
         Downloads the metadata for a media item.
@@ -472,7 +472,7 @@ def download_media_metadata(media_id):
              f'{source} / {media}: {media_id}')
 
 
-@background(schedule=dict(priority=15, run_at=10), remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=10), queue='network', remove_existing_tasks=True)
 def download_media_thumbnail(media_id, url):
     '''
         Downloads an image from a URL and save it as a local thumbnail attached to a
@@ -510,7 +510,7 @@ def download_media_thumbnail(media_id, url):
     return True
 
 
-@background(schedule=dict(priority=15, run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=60), queue='network', remove_existing_tasks=True)
 def download_media(media_id):
     '''
         Downloads the media to disk and attaches it to the Media instance.
@@ -632,7 +632,7 @@ def download_media(media_id):
         raise DownloadFailedException(err)
 
 
-@background(schedule=dict(priority=0, run_at=30), remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=30), queue='network', remove_existing_tasks=True)
 def rescan_media_server(mediaserver_id):
     '''
         Attempts to request a media rescan on a remote media server.
@@ -647,7 +647,7 @@ def rescan_media_server(mediaserver_id):
     mediaserver.update()
 
 
-@background(schedule=dict(priority=25, run_at=600), remove_existing_tasks=True)
+@background(schedule=dict(priority=25, run_at=600), queue='network', remove_existing_tasks=True)
 def save_all_media_for_source(source_id):
     '''
         Iterates all media items linked to a source and saves them to
@@ -704,7 +704,7 @@ def save_all_media_for_source(source_id):
     update_task_status(task, None)
 
 
-@background(schedule=dict(priority=20, run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), queue='filesystem', remove_existing_tasks=True)
 def rename_media(media_id):
     try:
         media = Media.objects.defer('metadata', 'thumb').get(pk=media_id)
@@ -713,7 +713,7 @@ def rename_media(media_id):
     media.rename_files()
 
 
-@background(schedule=dict(priority=20, run_at=300), remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=300), queue='filesystem', remove_existing_tasks=True)
 @atomic(durable=True)
 def rename_all_media_for_source(source_id):
     try:
@@ -746,7 +746,7 @@ def rename_all_media_for_source(source_id):
             media.rename_files()
 
 
-@background(schedule=dict(priority=0, run_at=60), remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=60), queue='database', remove_existing_tasks=True)
 def wait_for_media_premiere(media_id):
     hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
 
@@ -770,7 +770,7 @@ def wait_for_media_premiere(media_id):
         if task:
             update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=dict(priority=1, run_at=300), remove_existing_tasks=False)
+@background(schedule=dict(priority=1, run_at=300), queue='filesystem', remove_existing_tasks=False)
 def delete_all_media_for_source(source_id, source_name):
     source = None
     try:

From f316c3e81a728e5b983f579aedc99afa12ec3d61 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 19:28:05 -0400
Subject: [PATCH 37/49] Fix typo

---
 tubesync/sync/matching.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py
index 9390e6fa..ffb86416 100644
--- a/tubesync/sync/matching.py
+++ b/tubesync/sync/matching.py
@@ -236,7 +236,7 @@ def get_best_video_format(media):
                         break
             if not best_match:
                 for fmt in video_formats:
-                    # Check for codec and resolution match bot drop 60fps
+                    # Check for codec and resolution match but drop 60fps
                     if (source_resolution == fmt['format'] and
                         source_vcodec == fmt['vcodec'] and
                         not fmt['is_hdr']):
@@ -294,7 +294,7 @@ def get_best_video_format(media):
                         break
             if not best_match:
                 for fmt in video_formats:
-                    # Check for codec and resolution match bot drop hdr
+                    # Check for codec and resolution match but drop hdr
                     if (source_resolution == fmt['format'] and
                         source_vcodec == fmt['vcodec'] and
                         not fmt['is_60fps']):

From 73195fa79b0c873a5b798b9e03c95d86cda634ec Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 19:41:49 -0400
Subject: [PATCH 38/49] Do not use a thread pool for workers

---
 tubesync/tubesync/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py
index c44c888f..0ac2b462 100644
--- a/tubesync/tubesync/settings.py
+++ b/tubesync/tubesync/settings.py
@@ -136,7 +136,7 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
 
 MAX_ATTEMPTS = 15                           # Number of times tasks will be retried
 MAX_RUN_TIME = 1*(24*60*60)                 # Maximum amount of time in seconds a task can run
-BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
+BACKGROUND_TASK_RUN_ASYNC = False           # Run tasks async in the background
 BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at once
 MAX_BACKGROUND_TASK_ASYNC_THREADS = 8       # For sanity reasons
 BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering

From b33ff71678e1d4b64e88d9af2003bb6d1a555c8b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 20:38:26 -0400
Subject: [PATCH 39/49] Added additional tubesync workers

---
 .../{tubesync-worker => tubesync-db-worker}/dependencies     | 0
 .../{tubesync-worker => tubesync-db-worker}/down-signal      | 0
 config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run    | 5 +++++
 .../s6-rc.d/{tubesync-worker => tubesync-db-worker}/type     | 0
 .../etc/s6-overlay/s6-rc.d/tubesync-fs-worker/dependencies   | 1 +
 .../etc/s6-overlay/s6-rc.d/tubesync-fs-worker/down-signal    | 1 +
 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run    | 5 +++++
 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/type   | 1 +
 .../s6-overlay/s6-rc.d/tubesync-network-worker/dependencies  | 1 +
 .../s6-overlay/s6-rc.d/tubesync-network-worker/down-signal   | 1 +
 .../s6-rc.d/{tubesync-worker => tubesync-network-worker}/run | 0
 .../root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type | 1 +
 .../user/contents.d/{tubesync-worker => tubesync-db-worker}  | 0
 .../s6-overlay/s6-rc.d/user/contents.d/tubesync-fs-worker    | 0
 .../s6-rc.d/user/contents.d/tubesync-network-worker          | 0
 15 files changed, 16 insertions(+)
 rename config/root/etc/s6-overlay/s6-rc.d/{tubesync-worker => tubesync-db-worker}/dependencies (100%)
 rename config/root/etc/s6-overlay/s6-rc.d/{tubesync-worker => tubesync-db-worker}/down-signal (100%)
 create mode 100755 config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run
 rename config/root/etc/s6-overlay/s6-rc.d/{tubesync-worker => tubesync-db-worker}/type (100%)
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/dependencies
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/down-signal
 create mode 100755 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/type
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal
 rename config/root/etc/s6-overlay/s6-rc.d/{tubesync-worker => tubesync-network-worker}/run (100%)
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type
 rename config/root/etc/s6-overlay/s6-rc.d/user/contents.d/{tubesync-worker => tubesync-db-worker} (100%)
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-fs-worker
 create mode 100644 config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-network-worker

diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/dependencies
similarity index 100%
rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/dependencies
rename to config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/dependencies
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/down-signal b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/down-signal
similarity index 100%
rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/down-signal
rename to config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/down-signal
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run
new file mode 100755
index 00000000..03b75ea8
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/run
@@ -0,0 +1,5 @@
+#!/command/with-contenv bash
+
+exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py process_tasks \
+    --queue database
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/type b/config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/type
similarity index 100%
rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/type
rename to config/root/etc/s6-overlay/s6-rc.d/tubesync-db-worker/type
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/dependencies
new file mode 100644
index 00000000..283e1305
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/dependencies
@@ -0,0 +1 @@
+gunicorn
\ No newline at end of file
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/down-signal b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/down-signal
new file mode 100644
index 00000000..d751378e
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/down-signal
@@ -0,0 +1 @@
+SIGINT
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run
new file mode 100755
index 00000000..0642054d
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/run
@@ -0,0 +1,5 @@
+#!/command/with-contenv bash
+
+exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py process_tasks \
+    --queue filesystem
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/type b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/type
new file mode 100644
index 00000000..1780f9f4
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-fs-worker/type
@@ -0,0 +1 @@
+longrun
\ No newline at end of file
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies
new file mode 100644
index 00000000..283e1305
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies
@@ -0,0 +1 @@
+gunicorn
\ No newline at end of file
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal
new file mode 100644
index 00000000..d751378e
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal
@@ -0,0 +1 @@
+SIGINT
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run
similarity index 100%
rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run
rename to config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run
diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type
new file mode 100644
index 00000000..1780f9f4
--- /dev/null
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type
@@ -0,0 +1 @@
+longrun
\ No newline at end of file
diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-worker b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-db-worker
similarity index 100%
rename from config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-worker
rename to config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-db-worker
diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-fs-worker b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-fs-worker
new file mode 100644
index 00000000..e69de29b
diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-network-worker b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/tubesync-network-worker
new file mode 100644
index 00000000..e69de29b

From 6058a66df10e510c2722238211f1d474714a1fe6 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 20:44:23 -0400
Subject: [PATCH 40/49] Set executable bit on `full_playlist.sh`

---
 tubesync/full_playlist.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tubesync/full_playlist.sh

diff --git a/tubesync/full_playlist.sh b/tubesync/full_playlist.sh
old mode 100644
new mode 100755

From 4228d69023ac5eb389addca6ddd416f09be18074 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 20:48:57 -0400
Subject: [PATCH 41/49] Remove `queue` kwarg

---
 tubesync/sync/management/commands/reset-tasks.py | 1 -
 tubesync/sync/signals.py                         | 9 ---------
 tubesync/sync/views.py                           | 2 --
 3 files changed, 12 deletions(-)

diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py
index 3d5ecb98..d7818007 100644
--- a/tubesync/sync/management/commands/reset-tasks.py
+++ b/tubesync/sync/management/commands/reset-tasks.py
@@ -25,7 +25,6 @@ class Command(BaseCommand):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(source.pk),
-                queue=str(source.pk),
                 repeat=source.index_schedule,
                 verbose_name=verbose_name.format(source.name)
             )
diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 90b39480..6ee64747 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -92,7 +92,6 @@ def source_pre_save(sender, instance, **kwargs):
         verbose_name = _('Index media from source "{}"')
         index_source_task(
             str(instance.pk),
-            queue=str(instance.pk),
             repeat=instance.index_schedule,
             schedule=instance.index_schedule,
             verbose_name=verbose_name.format(instance.name),
@@ -106,13 +105,11 @@ def source_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Check download directory exists for source "{}"')
         check_source_directory_exists(
             str(instance.pk),
-            queue=str(instance.pk),
             verbose_name=verbose_name.format(instance.name),
         )
         if instance.source_type != Val(YouTube_SourceType.PLAYLIST) and instance.copy_channel_images:
             download_source_images(
                 str(instance.pk),
-                queue=str(instance.pk),
                 verbose_name=verbose_name.format(instance.name),
             )
         if instance.index_schedule > 0:
@@ -121,7 +118,6 @@ def source_post_save(sender, instance, created, **kwargs):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(instance.pk),
-                queue=str(instance.pk),
                 repeat=instance.index_schedule,
                 schedule=600,
                 verbose_name=verbose_name.format(instance.name),
@@ -130,7 +126,6 @@ def source_post_save(sender, instance, created, **kwargs):
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
-        queue=str(instance.pk),
         verbose_name=verbose_name.format(instance.name),
     )
 
@@ -238,7 +233,6 @@ def media_post_save(sender, instance, created, **kwargs):
             verbose_name = _('Renaming media for: {}: "{}"')
             rename_media(
                 str(media.pk),
-                queue=str(media.pk),
                 verbose_name=verbose_name.format(media.key, media.name),
             )
 
@@ -248,7 +242,6 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            queue=str(media.pk),
             verbose_name=verbose_name.format(instance.pk),
         )
     # If the media is missing a thumbnail schedule it to be downloaded (unless we are skipping this media)
@@ -263,7 +256,6 @@ def media_post_save(sender, instance, created, **kwargs):
             download_media_thumbnail(
                 str(instance.pk),
                 thumbnail_url,
-                queue=str(instance.pk),
                 verbose_name=verbose_name.format(instance.name),
             )
     # If the media has not yet been downloaded schedule it to be downloaded
@@ -278,7 +270,6 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading media for "{}"')
         download_media(
             str(instance.pk),
-            queue=str(instance.pk),
             verbose_name=verbose_name.format(instance.name),
         )
     # Save the instance if any changes were required
diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 5e937e5e..3f6eda84 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -122,7 +122,6 @@ class SourcesView(ListView):
             verbose_name = _('Index media from source "{}" once')
             index_source_task(
                 str(source.pk),
-                queue=str(source.pk),
                 remove_existing_tasks=False,
                 repeat=0,
                 schedule=30,
@@ -933,7 +932,6 @@ class ResetTasks(FormView):
             verbose_name = _('Index media from source "{}"')
             index_source_task(
                 str(source.pk),
-                queue=str(source.pk),
                 repeat=source.index_schedule,
                 verbose_name=verbose_name.format(source.name)
             )

From 2613d9392410f9ae475c70221be3ff52abee3b30 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Sun, 6 Apr 2025 20:50:11 -0400
Subject: [PATCH 42/49] Remove unneeded kwargs

---
 tubesync/sync/tasks.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index aec69216..651a02a8 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -434,12 +434,9 @@ def download_media_metadata(media_id):
                 verbose_name = _('Waiting for the premiere of "{}" at: {}')
                 wait_for_media_premiere(
                     str(media.pk),
-                    priority=0,
-                    queue=str(media.pk),
                     repeat=Task.HOURLY,
                     repeat_until = published_datetime + timedelta(hours=1),
                     verbose_name=verbose_name.format(media.key, published_datetime.isoformat(' ', 'seconds')),
-                    remove_existing_tasks=True,
                 )
                 raise_exception = False
         if raise_exception:

From b97de08ffdfb66f9c8a07d359e34d68108ad300b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 00:19:23 -0400
Subject: [PATCH 43/49] Filter on `task_params` instead of `queue`

---
 tubesync/sync/tasks.py | 8 ++++++--
 tubesync/sync/views.py | 6 ++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 651a02a8..d9610ddb 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -133,7 +133,7 @@ def get_source_completed_tasks(source_id, only_errors=False):
     '''
         Returns a queryset of CompletedTask objects for a source by source ID.
     '''
-    q = {'queue': source_id}
+    q = {'task_params__istartswith': f'[["{source_id}"'}
     if only_errors:
         q['failed_at__isnull'] = False
     return CompletedTask.objects.filter(**q).order_by('-failed_at')
@@ -167,7 +167,11 @@ def get_source_index_task(source_id):
 def delete_task_by_source(task_name, source_id):
     now = timezone.now()
     unlocked = Task.objects.unlocked(now)
-    return unlocked.filter(task_name=task_name, queue=str(source_id)).delete()
+    qs = unlocked.filter(
+        task_name=task_name,
+        task_params__istartswith=f'[["{source_id}"',
+    )
+    return qs.delete()
 
 
 def delete_task_by_media(task_name, args):
diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 3f6eda84..c9fee226 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -768,7 +768,8 @@ class TasksView(ListView):
     def get_queryset(self):
         qs = Task.objects.all()
         if self.filter_source:
-            qs = qs.filter(queue=str(self.filter_source.pk))
+            params_prefix=f'[["{self.filter_source.pk}"'
+            qs = qs.filter(task_params__istartswith=params_prefix)
         order = getattr(settings,
             'BACKGROUND_TASK_PRIORITY_ORDERING',
             'DESC'
@@ -896,7 +897,8 @@ class CompletedTasksView(ListView):
     def get_queryset(self):
         qs = CompletedTask.objects.all()
         if self.filter_source:
-            qs = qs.filter(queue=str(self.filter_source.pk))
+            params_prefix=f'[["{self.filter_source.pk}"'
+            qs = qs.filter(task_params__istartswith=params_prefix)
         return qs.order_by('-run_at')
 
     def get_context_data(self, *args, **kwargs):

From 8e52692ec89966af93d1be3199b196cf6469b32b Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 00:22:55 -0400
Subject: [PATCH 44/49] Label the queue as it is no longer `Source.uuid`

---
 tubesync/sync/templates/sync/tasks-completed.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubesync/sync/templates/sync/tasks-completed.html b/tubesync/sync/templates/sync/tasks-completed.html
index b87805be..52f576df 100644
--- a/tubesync/sync/templates/sync/tasks-completed.html
+++ b/tubesync/sync/templates/sync/tasks-completed.html
@@ -17,14 +17,14 @@
       {% if task.has_error %}
       <span class="collection-item">
         <i class="fas fa-exclamation-triangle"></i> <strong>{{ task.verbose_name }}</strong><br>
-        Source: &quot;{{ task.queue }}&quot;<br>
+        Queue: &quot;{{ task.queue }}&quot;<br>
         Error: &quot;{{ task.error_message }}&quot;<br>
         <i class="far fa-clock"></i> Task ran at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
       </span>
       {% else %}
       <span class="collection-item">
         <i class="fas fa-check"></i> <strong>{{ task.verbose_name }}</strong><br>
-        Source: &quot;{{ task.queue }}&quot;<br>
+        Queue: &quot;{{ task.queue }}&quot;<br>
         <i class="far fa-clock"></i> Task ran at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
       </span>
       {% endif %}

From 468242c626d1767629eb87a2d2db4fb1da861e87 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 00:29:14 -0400
Subject: [PATCH 45/49] Use the `TaskQueue` class

---
 tubesync/sync/choices.py |  6 ++++++
 tubesync/sync/tasks.py   | 25 +++++++++++++------------
 tubesync/sync/tests.py   |  4 ++--
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py
index c67de54b..25dd762a 100644
--- a/tubesync/sync/choices.py
+++ b/tubesync/sync/choices.py
@@ -160,6 +160,12 @@ class SponsorBlock_Category(models.TextChoices):
     MUSIC_OFFTOPIC = 'music_offtopic', _( 'Non-Music Section' )
 
 
+class TaskQueue(models.TextChoices):
+    DB = 'database', _('Database')
+    FS = 'filesystem', _('Filesystem')
+    NET = 'network', _('Networking')
+
+
 class YouTube_SourceType(models.TextChoices):
     CHANNEL = 'c', _('YouTube channel')
     CHANNEL_ID = 'i', _('YouTube channel by ID')
diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index d9610ddb..d937d690 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -26,6 +26,7 @@ from background_task.models import Task, CompletedTask
 from common.logger import log
 from common.errors import NoMediaException, NoMetadataException, DownloadFailedException
 from common.utils import json_serial, remove_enclosed
+from .choices import Val, TaskQueue
 from .models import Source, Media, MediaServer
 from .utils import (get_remote_image, resize_image_to_height, delete_file,
                     write_text_file, filter_response)
@@ -233,7 +234,7 @@ def cleanup_removed_media(source, videos):
     schedule_media_servers_update()
 
 
-@background(schedule=dict(priority=10, run_at=30), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=10, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def index_source_task(source_id):
     '''
         Indexes media available from a Source object.
@@ -320,7 +321,7 @@ def index_source_task(source_id):
     cleanup_removed_media(source, videos)
 
 
-@background(schedule=dict(priority=0, run_at=0), queue='filesystem')
+@background(schedule=dict(priority=0, run_at=0), queue=Val(TaskQueue.FS))
 def check_source_directory_exists(source_id):
     '''
         Checks the output directory for a source exists and is writable, if it does
@@ -339,7 +340,7 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
-@background(schedule=dict(priority=5, run_at=10), queue='network')
+@background(schedule=dict(priority=5, run_at=10), Val(TaskQueue.NET))
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a
@@ -389,7 +390,7 @@ def download_source_images(source_id):
     log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}')
 
 
-@background(schedule=dict(priority=20, run_at=60), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def download_media_metadata(media_id):
     '''
         Downloads the metadata for a media item.
@@ -473,7 +474,7 @@ def download_media_metadata(media_id):
              f'{source} / {media}: {media_id}')
 
 
-@background(schedule=dict(priority=15, run_at=10), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=10), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def download_media_thumbnail(media_id, url):
     '''
         Downloads an image from a URL and save it as a local thumbnail attached to a
@@ -511,7 +512,7 @@ def download_media_thumbnail(media_id, url):
     return True
 
 
-@background(schedule=dict(priority=15, run_at=60), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=15, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def download_media(media_id):
     '''
         Downloads the media to disk and attaches it to the Media instance.
@@ -633,7 +634,7 @@ def download_media(media_id):
         raise DownloadFailedException(err)
 
 
-@background(schedule=dict(priority=0, run_at=30), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def rescan_media_server(mediaserver_id):
     '''
         Attempts to request a media rescan on a remote media server.
@@ -648,7 +649,7 @@ def rescan_media_server(mediaserver_id):
     mediaserver.update()
 
 
-@background(schedule=dict(priority=25, run_at=600), queue='network', remove_existing_tasks=True)
+@background(schedule=dict(priority=25, run_at=600), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
 def save_all_media_for_source(source_id):
     '''
         Iterates all media items linked to a source and saves them to
@@ -705,7 +706,7 @@ def save_all_media_for_source(source_id):
     update_task_status(task, None)
 
 
-@background(schedule=dict(priority=20, run_at=60), queue='filesystem', remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=60), queue=Val(TaskQueue.FS), remove_existing_tasks=True)
 def rename_media(media_id):
     try:
         media = Media.objects.defer('metadata', 'thumb').get(pk=media_id)
@@ -714,7 +715,7 @@ def rename_media(media_id):
     media.rename_files()
 
 
-@background(schedule=dict(priority=20, run_at=300), queue='filesystem', remove_existing_tasks=True)
+@background(schedule=dict(priority=20, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=True)
 @atomic(durable=True)
 def rename_all_media_for_source(source_id):
     try:
@@ -747,7 +748,7 @@ def rename_all_media_for_source(source_id):
             media.rename_files()
 
 
-@background(schedule=dict(priority=0, run_at=60), queue='database', remove_existing_tasks=True)
+@background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.DB), remove_existing_tasks=True)
 def wait_for_media_premiere(media_id):
     hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
 
@@ -771,7 +772,7 @@ def wait_for_media_premiere(media_id):
         if task:
             update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=dict(priority=1, run_at=300), queue='filesystem', remove_existing_tasks=False)
+@background(schedule=dict(priority=1, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=False)
 def delete_all_media_for_source(source_id, source_name):
     source = None
     try:
diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index c16c4954..303aa18a 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -20,7 +20,7 @@ from .tasks import cleanup_old_media, check_source_directory_exists
 from .filtering import filter_media
 from .utils import filter_response
 from .choices import (Val, Fallback, IndexSchedule, SourceResolution,
-                        YouTube_AudioCodec, YouTube_VideoCodec,
+                        TaskQueue, YouTube_AudioCodec, YouTube_VideoCodec,
                         YouTube_SourceType, youtube_long_source_types)
 
 
@@ -211,7 +211,7 @@ class FrontEndTestCase(TestCase):
         source_uuid = str(source.pk)
         task = Task.objects.get_task('sync.tasks.index_source_task',
                                      args=(source_uuid,))[0]
-        self.assertEqual(task.queue, source_uuid)
+        self.assertEqual(task.queue, Val(TaskQueue.NET))
         # Run the check_source_directory_exists task
         check_source_directory_exists.now(source_uuid)
         # Check the source is now on the source overview page

From 10aa455ba9679f4e9d7ac0bb7549c16f0c8001ef Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 00:38:09 -0400
Subject: [PATCH 46/49] Limit the worker to a single queue

---
 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run
index b2c3a841..a9c17d49 100755
--- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run
+++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run
@@ -1,4 +1,5 @@
 #!/command/with-contenv bash
 
 exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
-    /usr/bin/python3 /app/manage.py process_tasks
+    /usr/bin/python3 /app/manage.py process_tasks \
+    --queue network

From e2b99de843d5bdca12c8a71ae4fb9b05f53b237c Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 01:26:52 -0400
Subject: [PATCH 47/49] fixup: restore the missing keyword

---
 tubesync/sync/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index d937d690..7c10c038 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -340,7 +340,7 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
-@background(schedule=dict(priority=5, run_at=10), Val(TaskQueue.NET))
+@background(schedule=dict(priority=5, run_at=10), queue=Val(TaskQueue.NET))
 def download_source_images(source_id):
     '''
         Downloads an image and save it as a local thumbnail attached to a

From 0c056cc115c8ce092a65252b5ae1b5671bed8dbb Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 03:51:33 -0400
Subject: [PATCH 48/49] Add `migrate_queues` function

---
 tubesync/sync/tasks.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 7c10c038..34c37e6c 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -192,6 +192,13 @@ def cleanup_completed_tasks():
     CompletedTask.objects.filter(run_at__lt=delta).delete()
 
 
+@atomic(durable=False)
+def migrate_queues():
+    tqs = Task.objects.all()
+    qs = tqs.exclude(queue__in=TaskQueue.values)
+    return qs.update(queue=Val(TaskQueue.NET))
+
+
 def schedule_media_servers_update():
     with atomic():
         # Schedule a task to update media servers

From 2c41a90695ec76984abb2ee759aa458012c688b9 Mon Sep 17 00:00:00 2001
From: tcely <tcely@users.noreply.github.com>
Date: Mon, 7 Apr 2025 03:57:31 -0400
Subject: [PATCH 49/49] Migrate old tasks to the new queues

---
 tubesync/sync/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index c9fee226..f489144b 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -29,7 +29,7 @@ from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMedia
 from .utils import validate_url, delete_file, multi_key_sort
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
-                    delete_task_by_media, index_source_task)
+                    delete_task_by_media, index_source_task, migrate_queues)
 from .choices import (Val, MediaServerType, SourceResolution,
                         YouTube_SourceType, youtube_long_source_types,
                         youtube_help, youtube_validation_urls)
@@ -797,6 +797,7 @@ class TasksView(ListView):
         data['total_errors'] = errors_qs.count()
         data['scheduled'] = list()
         data['total_scheduled'] = scheduled_qs.count()
+        data['migrated'] = migrate_queues()
 
         def add_to_task(task):
             obj, url = map_task_to_instance(task)