Merge pull request #886 from tcely/patch-6

Force new `yt-dlp` version

.github/actions/FFmpeg/action.yml

@@ -0,0 +1,80 @@
+name: 'FFmpeg Builds'
+description: 'Use GitHub CLI & API to retrieve information about FFmpeg Build releases.'
+
+inputs:
+  token:
+    required: true
+    default: ${{ github.token }}
+    description: |
+      GH_TOKEN for GitHub CLI to use.
+      Default: `\$\{\{ github.token \}\}`
+  num-assets:
+    required: true
+    default: '25'
+    description: |
+      The number of assets (attached files) to retrieve from each release.
+      Default: 25
+  num-releases:
+    required: true
+    default: '35'
+    description: |
+      The number of releases to retrieve from the repository.
+      Default: 35
+  repository_owner:
+    required: true
+    default: 'yt-dlp'
+    description: |
+      The name of the user or organization that owns the repository.
+      Default: 'yt-dlp'
+  repository_name:
+    required: true
+    default: 'FFmpeg-Builds'
+    description: |
+      Which repository from the owner to search for releases.
+      Default: 'FFmpeg-Builds'
+
+outputs:
+  releases:
+    value: ${{ steps.set.outputs.releases }}
+    description: 'Generated JSON describing the released builds.'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Retrieve releases
+      id: 'set'
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        GH_API_GQL_ASSETS: '${{ inputs.num-assets }}'
+        GH_API_GQL_RELEASES: '${{ inputs.num-releases }}'
+        GH_API_GQL_OWNER: '${{ inputs.repository_owner }}'
+        GH_API_GQL_REPO: '${{ inputs.repository_name }}'
+      shell: 'bash'
+      run: |
+        command -v gh
+        command -v jq
+        gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ;
+        gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ;
+        mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ;
+        open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+        close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+        {
+          var='releases' ;
+          delim="$(mk_delim "${var}")" ;
+          open_ml_var "${delim}" "${var}" ;
+          gh api graphql --cache 12h \
+            -F assets="${GH_API_GQL_ASSETS}" \
+            -F owner="${GH_API_GQL_OWNER}" \
+            -F repo="${GH_API_GQL_REPO}" \
+            -F releases="${GH_API_GQL_RELEASES}" \
+            -f query="${gql_query}" --jq "${gql_jq}" ;
+          close_ml_var "${delim}" "${var}" ;
+          unset -v delim jq_arg var ;
+        } >> "${GITHUB_OUTPUT}" ;
+        # Log the human version
+        gh api graphql --cache 12h \
+          -F assets="${GH_API_GQL_ASSETS}" \
+          -F owner="${GH_API_GQL_OWNER}" \
+          -F repo="${GH_API_GQL_REPO}" \
+          -F releases="${GH_API_GQL_RELEASES}" \
+          -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ;

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: Run Django tests for TubeSync
+name: CI
 
 env:
   IMAGE_NAME: tubesync
@@ -8,9 +8,95 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
 
 jobs:
+  info:
+    #if: ${{ !cancelled() && 'pull_request' != github.event_name }}
+    runs-on: ubuntu-latest
+    outputs:
+      ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }}
+      lowercase-variables-actor: ${{ steps.lowercase-variables.outputs.actor }}
+      lowercase-variables-repository_owner: ${{ steps.lowercase-variables.outputs.repository_owner }}
+      string-lowercase: ${{ steps.string.outputs.lowercase }}
+      ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }}
+      ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }}
+    steps:
+      - name: Lowercase github username for ghcr
+        id: string
+        uses: ASzc/change-string-case-action@v6
+        with:
+          string: ${{ github.actor }}
+      - name: Lowercase GitHub variables
+        id: lowercase-variables
+        shell: bash
+        run: |
+          set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
+          set -x ; for var in \
+            actor='${{ github.actor }}' \
+            repository_owner='${{ github.repository_owner }}'
+          do
+            k="$( cut -d '=' -f 1 <<<"${var}" )" ;
+            v="${var#${k}=}" ;
+            set_sl_var >> "${GITHUB_OUTPUT}" \
+              "${k}" "${v,,}" ;
+          done ;
+          unset -v k v var ;
+      - uses: actions/checkout@v4
+      - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI
+        id: ffmpeg
+        uses: ./.github/actions/FFmpeg
+      - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI
+        id: yt-dlp
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_API_GQL_RELEASES: 25
+          GH_API_GQL_OWNER: yt-dlp
+          GH_API_GQL_REPO: yt-dlp
+        run: |
+          gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, tagName, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ;
+          gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ;
+          mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            var='releases' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            gh api graphql --cache 12h \
+              -F owner="${GH_API_GQL_OWNER}" \
+              -F repo="${GH_API_GQL_REPO}" \
+              -F releases="${GH_API_GQL_RELEASES}" \
+              -f query="${gql_query}" --jq "${gql_jq}" ;
+            close_ml_var "${delim}" "${var}" ;
+            jq_arg='map(select(.isLatest))[0]' ;
+            var='latest-release' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            gh api graphql --cache 12h \
+              -F owner="${GH_API_GQL_OWNER}" \
+              -F repo="${GH_API_GQL_REPO}" \
+              -F releases="${GH_API_GQL_RELEASES}" \
+              -f query="${gql_query}" --jq "${gql_jq}" | jq -c "${jq_arg}" -- ;
+            close_ml_var "${delim}" "${var}" ;
+            unset -v delim jq_arg var ;
+          } >> "${GITHUB_OUTPUT}" ;
+          # Log the human version
+          gh api graphql --cache 12h \
+            -F owner="${GH_API_GQL_OWNER}" \
+            -F repo="${GH_API_GQL_REPO}" \
+            -F releases="${GH_API_GQL_RELEASES}" \
+            -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ;
+
   test:
+    if: ${{ !cancelled() && ( 'pull_request' != github.event_name || (! github.event.pull_request.draft) ) }}
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
@@ -34,63 +120,37 @@ jobs:
           cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/*
       - name: Run Django tests
         run: cd tubesync && python3 manage.py test --verbosity=2
+
   containerise:
-    if: ${{ !cancelled() }}
+    if: ${{ !cancelled() && 'success' == needs.info.result }}
-    needs: test
+    needs: ['info', 'test']
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI
-        id: ffmpeg
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_API_GQL_ASSETS: 25
-          GH_API_GQL_RELEASES: 35
-          GH_API_GQL_OWNER: yt-dlp
-          GH_API_GQL_REPO: FFmpeg-Builds
-        run: |
-          gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ;
-          gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ;
-          {
-            var='releases' ;
-            delim='"'"${var}"'_EOF"' ;
-            printf -- '%s<<%s\n' "${var}" "${delim}" ;
-            gh api graphql --cache 12h \
-              -F assets="${GH_API_GQL_ASSETS}" \
-              -F owner="${GH_API_GQL_OWNER}" \
-              -F repo="${GH_API_GQL_REPO}" \
-              -F releases="${GH_API_GQL_RELEASES}" \
-              -f query="${gql_query}" --jq "${gql_jq}" ;
-            printf -- '%s\n' "${delim}" ;
-            unset -v delim jq_arg var ;
-          } >> "${GITHUB_OUTPUT}"
-            gh api graphql --cache 12h \
-              -F assets="${GH_API_GQL_ASSETS}" \
-              -F owner="${GH_API_GQL_OWNER}" \
-              -F repo="${GH_API_GQL_REPO}" \
-              -F releases="${GH_API_GQL_RELEASES}" \
-              -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ;
       - name: Set environment variables with jq
         run: |
           cat >| .ffmpeg.releases.json <<'EOF'
-          ${{ steps.ffmpeg.outputs.releases }}
+          ${{ needs.info.outputs.ffmpeg-releases }}
           EOF
+          mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
           {
             var='FFMPEG_DATE' ;
-            delim='"'"${var}"'_EOF"' ;
+            delim="$(mk_delim "${var}")" ;
-            printf -- '%s<<%s\n' "${var}" "${delim}" ;
+            open_ml_var "${delim}" "${var}" ;
             jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
             jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
-            printf -- '%s\n' "${delim}" ;
+            close_ml_var "${delim}" "${var}" ;
 
             ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
             
             var='FFMPEG_VERSION' ;
-            delim='"'"${var}"'_EOF"' ;
+            delim="$(mk_delim "${var}")" ;
-            printf -- '%s<<%s\n' "${var}" "${delim}" ;
+            open_ml_var "${delim}" "${var}" ;
             jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
             jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
-            printf -- '%s\n' "${delim}" ;
+            close_ml_var "${delim}" "${var}" ;
             unset -v delim jq_arg var ;
           } >> "${GITHUB_ENV}"
       - name: Set up QEMU
@@ -104,21 +164,22 @@ jobs:
           DOCKER_USERNAME: ${{ github.actor }}
           DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
         run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}"
-      - name: Lowercase github username for ghcr
-        id: string
-        uses: ASzc/change-string-case-action@v6
-        with:
-          string: ${{ github.actor }}
       - name: Build and push
         timeout-minutes: 60
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
-          push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'true' || 'false' }}
+          push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }}
-          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          tags: ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest
-          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          cache-from: |
-          cache-to: type=inline
+            type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest
+            type=gha
+          cache-to: |
+            type=gha,mode=max
+            ${{ 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'type=inline' || '' }}
           build-args: |
             IMAGE_NAME=${{ env.IMAGE_NAME }}
             FFMPEG_DATE=${{ env.FFMPEG_DATE }}
             FFMPEG_VERSION=${{ env.FFMPEG_VERSION }}
+            YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}

Dockerfile

@@ -315,6 +315,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
 # Switch workdir to the the app
 WORKDIR /app
 
+ARG YTDLP_DATE
+
 # Set up the app
 RUN --mount=type=tmpfs,target=/cache \
     --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \