From f150ce21d0829653dc1cb4c20e2c2c4a8ef76d4a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 02:44:58 -0400 Subject: [PATCH 01/28] Force new `yt-dlp` version --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c0fcd9ad..82bc665e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -315,6 +315,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va # Switch workdir to the the app WORKDIR /app +ARG YTDLP_DATE + # Set up the app RUN --mount=type=tmpfs,target=/cache \ --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \ From 6e38fcdb7c184e79f8d106d97d4ba9336d3248a0 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 02:49:35 -0400 Subject: [PATCH 02/28] Create `tag-yt-dlp` --- tag-yt-dlp | 1 + 1 file changed, 1 insertion(+) create mode 100644 tag-yt-dlp diff --git a/tag-yt-dlp b/tag-yt-dlp new file mode 100644 index 00000000..0c159067 --- /dev/null +++ b/tag-yt-dlp @@ -0,0 +1 @@ + 2025.03.26 From 5e6c9f144a37d191518c65ed196b21d1cd6cda85 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 03:06:20 -0400 Subject: [PATCH 03/28] Add `YTDLP_DATE` build arg --- .github/workflows/ci.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7eef6ea..24d66353 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,18 @@ on: - main jobs: + info: + runs-on: ubuntu-latest + outputs: + ytdlp_tag: ${{ steps.set.outputs.ytdlp_tag }} + steps: + - uses: actions/checkout@v4 + - name: Set outputs + id: set + run: | + printf -- '%s=%s\n' >> "${GITHUB_OUTPUT}" \ + 'ytdlp_tag' "$(< tag-yt-dlp)" + test: runs-on: ubuntu-22.04 strategy: @@ -34,9 +46,10 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 + containerise: if: ${{ !cancelled() }} - needs: test + needs: ['info', 'test'] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -122,3 +135,5 @@ jobs: IMAGE_NAME=${{ env.IMAGE_NAME }} FFMPEG_DATE=${{ env.FFMPEG_DATE }} FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} + YTDLP_DATE=${{ needs.info.outputs.ytdlp_tag }} + From cacf306b0960b8cf3b66f15316d0e5ba3b0b4902 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 03:30:13 -0400 Subject: [PATCH 04/28] Move more steps into the `info` job --- .github/workflows/ci.yaml | 79 ++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 24d66353..30220cc7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,8 +13,45 @@ jobs: info: runs-on: ubuntu-latest outputs: + ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} + string-lowercase: ${{ steps.string.outputs.lowercase }} ytdlp_tag: ${{ steps.set.outputs.ytdlp_tag }} steps: + - name: Lowercase github username for ghcr + id: string + uses: ASzc/change-string-case-action@v6 + with: + string: ${{ github.actor }} + - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI + id: ffmpeg + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_API_GQL_ASSETS: 25 + GH_API_GQL_RELEASES: 35 + GH_API_GQL_OWNER: yt-dlp + GH_API_GQL_REPO: FFmpeg-Builds + run: | + gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; + { + var='releases' ; + delim='"'"${var}"'_EOF"' ; + printf -- '%s<<%s\n' "${var}" "${delim}" ; + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" ; + printf -- '%s\n' "${delim}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - uses: actions/checkout@v4 - name: Set outputs id: set @@ -53,40 +90,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: - - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI - id: ffmpeg - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_API_GQL_ASSETS: 25 - GH_API_GQL_RELEASES: 35 - GH_API_GQL_OWNER: yt-dlp - GH_API_GQL_REPO: FFmpeg-Builds - run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; - gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; - { - var='releases' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" ; - printf -- '%s\n' "${delim}" ; - unset -v delim jq_arg var ; - } >> "${GITHUB_OUTPUT}" - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - name: Set environment variables with jq run: | cat >| .ffmpeg.releases.json <<'EOF' - ${{ steps.ffmpeg.outputs.releases }} + ${{ needs.info.outputs.ffmpeg-releases }} EOF { var='FFMPEG_DATE' ; @@ -117,23 +124,17 @@ jobs: DOCKER_USERNAME: ${{ github.actor }} DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}" - - name: Lowercase github username for ghcr - id: string - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.actor }} - name: Build and push timeout-minutes: 60 uses: docker/build-push-action@v6 with: platforms: linux/amd64,linux/arm64 push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'true' || 'false' }} - tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest + tags: ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest + cache-from: type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest cache-to: type=inline build-args: | IMAGE_NAME=${{ env.IMAGE_NAME }} FFMPEG_DATE=${{ env.FFMPEG_DATE }} FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} YTDLP_DATE=${{ needs.info.outputs.ytdlp_tag }} - From b799bdfd65977513b9e9e8910ec7ce173d826396 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 04:05:21 -0400 Subject: [PATCH 05/28] Update tag-yt-dlp --- tag-yt-dlp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tag-yt-dlp b/tag-yt-dlp index 0c159067..c7302ddb 100644 --- a/tag-yt-dlp +++ b/tag-yt-dlp @@ -1 +1 @@ - 2025.03.26 +2025.03.26 From 0300be4728f7d893816be4b1d81d37acab634140 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 12:23:52 -0400 Subject: [PATCH 06/28] Retrieve `yt-dlp/yt-dlp` releases from the GitHub API --- .github/workflows/ci.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 30220cc7..b445aec5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,6 +52,33 @@ jobs: -F repo="${GH_API_GQL_REPO}" \ -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; + - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI + id: yt-dlp + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_API_GQL_RELEASES: 25 + GH_API_GQL_OWNER: yt-dlp + GH_API_GQL_REPO: yt-dlp + run: | + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | { "tag": .tag, "date": .tag.name } ]' ; + { + var='releases' ; + delim='"'"${var}"'_EOF"' ; + printf -- '%s<<%s\n' "${var}" "${delim}" ; + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" ; + printf -- '%s\n' "${delim}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - uses: actions/checkout@v4 - name: Set outputs id: set From 18829d13628c33a0427b2d63a43c4894028ebe42 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 12:49:04 -0400 Subject: [PATCH 07/28] Show the raw API response --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b445aec5..2546ab0c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -60,8 +60,8 @@ jobs: GH_API_GQL_OWNER: yt-dlp GH_API_GQL_REPO: yt-dlp run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; - gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | { "tag": .tag, "date": .tag.name } ]' ; + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) ]' ; { var='releases' ; delim='"'"${var}"'_EOF"' ; From 72ba0474a38f0621b1fc52384b4248bb94ee3077 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 13:10:22 -0400 Subject: [PATCH 08/28] Remove static booleans --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2546ab0c..30ad207b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -60,8 +60,8 @@ jobs: GH_API_GQL_OWNER: yt-dlp GH_API_GQL_REPO: yt-dlp run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; - gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) ]' ; + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, description, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ; { var='releases' ; delim='"'"${var}"'_EOF"' ; From 085c64d95ddcb0e638d55778ee1ae35af039aea3 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 14:08:07 -0400 Subject: [PATCH 09/28] Show the `yt-dlp-latest-release` --- .github/workflows/ci.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 30ad207b..2e241a7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,6 +15,7 @@ jobs: outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} string-lowercase: ${{ steps.string.outputs.lowercase }} + ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} ytdlp_tag: ${{ steps.set.outputs.ytdlp_tag }} steps: - name: Lowercase github username for ghcr @@ -60,11 +61,12 @@ jobs: GH_API_GQL_OWNER: yt-dlp GH_API_GQL_REPO: yt-dlp run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, description, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ; { var='releases' ; - delim='"'"${var}"'_EOF"' ; + delim="$(mk_delim "${var}")" ; printf -- '%s<<%s\n' "${var}" "${delim}" ; gh api graphql --cache 12h \ -F owner="${GH_API_GQL_OWNER}" \ @@ -72,6 +74,16 @@ jobs: -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" ; printf -- '%s\n' "${delim}" ; + jq_arg='map(select(.isLatest))' ; + var='latest-release' ; + delim="$(mk_delim "${var}")" ; + printf -- '%s<<%s\n' "${var}" "${delim}" ; + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq -c "${jq_arg}" -- ; + printf -- '%s\n' "${delim}" ; unset -v delim jq_arg var ; } >> "${GITHUB_OUTPUT}" gh api graphql --cache 12h \ @@ -83,6 +95,9 @@ jobs: - name: Set outputs id: set run: | + cat <<'EOF' + ${{ toJSON(fromJSON(steps.yt-dlp.outputs.latest-release)) }} + 'EOF' printf -- '%s=%s\n' >> "${GITHUB_OUTPUT}" \ 'ytdlp_tag' "$(< tag-yt-dlp)" From 77d00114f83c8ade466d10759ae5c9d98724aee2 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 14:16:25 -0400 Subject: [PATCH 10/28] Show the tag name from the latest `yt-dlp` release --- .github/workflows/ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2e241a7b..2aff0e9a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,7 +74,7 @@ jobs: -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" ; printf -- '%s\n' "${delim}" ; - jq_arg='map(select(.isLatest))' ; + jq_arg='map(select(.isLatest))[0]' ; var='latest-release' ; delim="$(mk_delim "${var}")" ; printf -- '%s<<%s\n' "${var}" "${delim}" ; @@ -97,7 +97,10 @@ jobs: run: | cat <<'EOF' ${{ toJSON(fromJSON(steps.yt-dlp.outputs.latest-release)) }} - 'EOF' + EOF + cat <<'EOF' + ${{ fromJSON(steps.yt-dlp.outputs.latest-release).tag.name }} + EOF printf -- '%s=%s\n' >> "${GITHUB_OUTPUT}" \ 'ytdlp_tag' "$(< tag-yt-dlp)" From ff502e2014161360d78c1495fb941d2a7a82caa0 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 14:40:20 -0400 Subject: [PATCH 11/28] Clean up the `set` step --- .github/workflows/ci.yaml | 53 ++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2aff0e9a..5b7d0562 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ jobs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} string-lowercase: ${{ steps.string.outputs.lowercase }} ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} - ytdlp_tag: ${{ steps.set.outputs.ytdlp_tag }} + ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }} steps: - name: Lowercase github username for ghcr id: string @@ -32,11 +32,12 @@ jobs: GH_API_GQL_OWNER: yt-dlp GH_API_GQL_REPO: FFmpeg-Builds run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; + gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; { var='releases' ; - delim='"'"${var}"'_EOF"' ; + delim="$(mk_delim "${var}")" ; printf -- '%s<<%s\n' "${var}" "${delim}" ; gh api graphql --cache 12h \ -F assets="${GH_API_GQL_ASSETS}" \ @@ -46,13 +47,14 @@ jobs: -f query="${gql_query}" --jq "${gql_jq}" ; printf -- '%s\n' "${delim}" ; unset -v delim jq_arg var ; - } >> "${GITHUB_OUTPUT}" - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; + } >> "${GITHUB_OUTPUT}" ; + # Log the human version + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI id: yt-dlp env: @@ -61,9 +63,9 @@ jobs: GH_API_GQL_OWNER: yt-dlp GH_API_GQL_REPO: yt-dlp run: | - mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; - gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, tagName, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; { var='releases' ; delim="$(mk_delim "${var}")" ; @@ -85,24 +87,13 @@ jobs: -f query="${gql_query}" --jq "${gql_jq}" | jq -c "${jq_arg}" -- ; printf -- '%s\n' "${delim}" ; unset -v delim jq_arg var ; - } >> "${GITHUB_OUTPUT}" - gh api graphql --cache 12h \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - - uses: actions/checkout@v4 - - name: Set outputs - id: set - run: | - cat <<'EOF' - ${{ toJSON(fromJSON(steps.yt-dlp.outputs.latest-release)) }} - EOF - cat <<'EOF' - ${{ fromJSON(steps.yt-dlp.outputs.latest-release).tag.name }} - EOF - printf -- '%s=%s\n' >> "${GITHUB_OUTPUT}" \ - 'ytdlp_tag' "$(< tag-yt-dlp)" + } >> "${GITHUB_OUTPUT}" ; + # Log the human version + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; test: runs-on: ubuntu-22.04 @@ -182,4 +173,4 @@ jobs: IMAGE_NAME=${{ env.IMAGE_NAME }} FFMPEG_DATE=${{ env.FFMPEG_DATE }} FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} - YTDLP_DATE=${{ needs.info.outputs.ytdlp_tag }} + YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} From 63a65949c7688ed977d3b1f0676a4c2c49936d11 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 14:43:54 -0400 Subject: [PATCH 12/28] Delete `tag-yt-dlp` as no longer used --- tag-yt-dlp | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tag-yt-dlp diff --git a/tag-yt-dlp b/tag-yt-dlp deleted file mode 100644 index c7302ddb..00000000 --- a/tag-yt-dlp +++ /dev/null @@ -1 +0,0 @@ -2025.03.26 From af4374c224733f4737f8adc9170e7e9919824cd4 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 15:26:38 -0400 Subject: [PATCH 13/28] Run tests for pull requests --- .github/workflows/ci.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5b7d0562..bcee7dc7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,9 +8,18 @@ on: push: branches: - main + pull_request: + branches: + - main + types: + - opened + - reopened + - synchronize + - ready_for_review jobs: info: + if: ${{ !cancelled() && 'pull_request' != github.event_name }} runs-on: ubuntu-latest outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} @@ -96,6 +105,7 @@ jobs: -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; test: + if: ${{ !cancelled() && ( 'pull_request' != github.event_name || (! github.event.pull_request.draft) ) }} runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -121,7 +131,7 @@ jobs: run: cd tubesync && python3 manage.py test --verbosity=2 containerise: - if: ${{ !cancelled() }} + if: ${{ !cancelled() && 'success' == needs.info.result }} needs: ['info', 'test'] runs-on: ubuntu-latest timeout-minutes: 120 From 952c2e530f09c58f0310a4b31f47c37471e2759d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 15:36:17 -0400 Subject: [PATCH 14/28] Shorten the name for checks --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcee7dc7..914c2556 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Run Django tests for TubeSync +name: CI env: IMAGE_NAME: tubesync From 807d825904d3ce48ebc575c96c75c2f5cbb98d43 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 15:52:33 -0400 Subject: [PATCH 15/28] Disable push for pull requests --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 914c2556..52742835 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ on: jobs: info: - if: ${{ !cancelled() && 'pull_request' != github.event_name }} + #if: ${{ !cancelled() && 'pull_request' != github.event_name }} runs-on: ubuntu-latest outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} @@ -175,7 +175,7 @@ jobs: uses: docker/build-push-action@v6 with: platforms: linux/amd64,linux/arm64 - push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'true' || 'false' }} + push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }} tags: ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest cache-from: type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest cache-to: type=inline From 317cc63a96f7931f8e4acecf3e778cbe506f1e55 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 16:45:15 -0400 Subject: [PATCH 16/28] Look for docker layers in multiple cache locations --- .github/workflows/ci.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 52742835..fb6934be 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -177,8 +177,13 @@ jobs: platforms: linux/amd64,linux/arm64 push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }} tags: ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-to: type=inline + cache-from: | + type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest + type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest + type=gha + cache-to: | + type=gha,mode=max + ${{ 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'type=inline' || '' }} build-args: | IMAGE_NAME=${{ env.IMAGE_NAME }} FFMPEG_DATE=${{ env.FFMPEG_DATE }} From ea6beae40ae351e89ec53fb2073579032b1b8c1f Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 16:58:44 -0400 Subject: [PATCH 17/28] Turn off the `info` step again for pull requests --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb6934be..92b72d1e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ on: jobs: info: - #if: ${{ !cancelled() && 'pull_request' != github.event_name }} + if: ${{ !cancelled() && 'pull_request' != github.event_name }} runs-on: ubuntu-latest outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} From 1885aecb4b23a86e33d7e0c0e60ebfc9f8162888 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 17:29:41 -0400 Subject: [PATCH 18/28] Lowercase some variables using bash --- .github/workflows/ci.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92b72d1e..08f2ec63 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,6 +23,8 @@ jobs: runs-on: ubuntu-latest outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} + lowercase-variables-actor: ${{ steps.lowercase-variables.outputs.actor }} + lowercase-variables-repository_owner: ${{ steps.lowercase-variables.outputs.repository_owner }} string-lowercase: ${{ steps.string.outputs.lowercase }} ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }} @@ -32,6 +34,20 @@ jobs: uses: ASzc/change-string-case-action@v6 with: string: ${{ github.actor }} + - name: Lowercase GitHub variables + id: lowercase-variables + shell: bash + run: | + for var in \ + actor='${{ github.actor }}' \ + repository_owner='${{ github.repository_owner }}' + do + k="$(cut -d '=' -f 1)" + v="${var#${k}=}" + printf -- '%s=%s\n' -- >> "${GITHUB_OUTPUT}" \ + "${k}" "${v,,}" + done + unset -v k v var - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI id: ffmpeg env: From d22f200b732b5c4a903625451779b966952884f5 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 20:06:50 -0400 Subject: [PATCH 19/28] Improve readability Also, work around a strange syntax highlighting behavior. --- .github/workflows/ci.yaml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08f2ec63..af3066c9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -60,17 +60,19 @@ jobs: gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; { var='releases' ; delim="$(mk_delim "${var}")" ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + open_ml_var "${delim}" "${var}" ; gh api graphql --cache 12h \ -F assets="${GH_API_GQL_ASSETS}" \ -F owner="${GH_API_GQL_OWNER}" \ -F repo="${GH_API_GQL_REPO}" \ -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; unset -v delim jq_arg var ; } >> "${GITHUB_OUTPUT}" ; # Log the human version @@ -91,26 +93,28 @@ jobs: gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, tagName, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ; mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; { var='releases' ; delim="$(mk_delim "${var}")" ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + open_ml_var "${delim}" "${var}" ; gh api graphql --cache 12h \ -F owner="${GH_API_GQL_OWNER}" \ -F repo="${GH_API_GQL_REPO}" \ -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; jq_arg='map(select(.isLatest))[0]' ; var='latest-release' ; delim="$(mk_delim "${var}")" ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + open_ml_var "${delim}" "${var}" ; gh api graphql --cache 12h \ -F owner="${GH_API_GQL_OWNER}" \ -F repo="${GH_API_GQL_REPO}" \ -F releases="${GH_API_GQL_RELEASES}" \ -f query="${gql_query}" --jq "${gql_jq}" | jq -c "${jq_arg}" -- ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; unset -v delim jq_arg var ; } >> "${GITHUB_OUTPUT}" ; # Log the human version @@ -157,22 +161,25 @@ jobs: cat >| .ffmpeg.releases.json <<'EOF' ${{ needs.info.outputs.ffmpeg-releases }} EOF + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; { var='FFMPEG_DATE' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ; jq -r "${jq_arg}" -- .ffmpeg.releases.json ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )" var='FFMPEG_VERSION' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ; jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; unset -v delim jq_arg var ; } >> "${GITHUB_ENV}" - name: Set up QEMU From dc49b6d87c2c2cc78f237613f8bdc61a0a1985c2 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 20:17:16 -0400 Subject: [PATCH 20/28] Add a function for single line variables also --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af3066c9..e395239c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,13 +38,14 @@ jobs: id: lowercase-variables shell: bash run: | + set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ; for var in \ actor='${{ github.actor }}' \ repository_owner='${{ github.repository_owner }}' do k="$(cut -d '=' -f 1)" v="${var#${k}=}" - printf -- '%s=%s\n' -- >> "${GITHUB_OUTPUT}" \ + set_sl_var >> "${GITHUB_OUTPUT}" \ "${k}" "${v,,}" done unset -v k v var From 3a9ff40ddab5b80d79ed95e50e25a28894ad75cc Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 21:10:54 -0400 Subject: [PATCH 21/28] Create action.yml This is only a sketch of moving the API work into reusable actions so far. --- .github/actions/FFmpeg/action.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/actions/FFmpeg/action.yml diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml new file mode 100644 index 00000000..ed52cd12 --- /dev/null +++ b/.github/actions/FFmpeg/action.yml @@ -0,0 +1,27 @@ +name: 'FFmpeg Builds' +description: 'Use GitHub CLI & API to retrieve information about FFmpeg Build releases.' + +inputs: + who-to-greet: # id of input + description: 'Who to greet' + required: false + default: 'World' + +outputs: + random-number: + description: "Random number" + value: ${{ steps.first.outputs.random-number }} + +runs: + using: 'composite' + steps: + - name: First step + id: 'first' + env: + INPUT_WHO_TO_GREET: ${{ inputs.who-to-greet }} + shell: 'bash' + run: | + echo 'It worked!' + echo "random-number=${RANDOM}" >> "${GITHUB_OUTPUT}" + ls -al '${{ github.action_path }}' + echo "Hello ${INPUT_WHO_TO_GREET}." From e473af2878aaf9a53f729562000ff2fc070eec6e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 21:50:16 -0400 Subject: [PATCH 22/28] Replace the sketch with the current code --- .github/actions/FFmpeg/action.yml | 81 +++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml index ed52cd12..90a1210f 100644 --- a/.github/actions/FFmpeg/action.yml +++ b/.github/actions/FFmpeg/action.yml @@ -2,26 +2,79 @@ name: 'FFmpeg Builds' description: 'Use GitHub CLI & API to retrieve information about FFmpeg Build releases.' inputs: - who-to-greet: # id of input - description: 'Who to greet' - required: false - default: 'World' + token: + required: true + default: ${{ secrets.GITHUB_TOKEN }} + description: | + GH_TOKEN for GitHub CLI to use. + Default: $${{ secrets.GITHUB_TOKEN }} + num-assets: + required: true + default: '25' + description: | + The number of assets (attached files) to retrieve from each release. + Default: 25 + num-releases: + required: true + default: '35' + description: | + The number of releases to retrieve from the repository. + Default: 35 + repository_owner: + required: true + default: 'yt-dlp' + description: | + The name of the user or organization that owns the repository. + Default: 'yt-dlp' + repository_name: + required: true + default: 'FFmpeg-Builds' + description: | + Which repository from the owner to search for releases. + Default: 'FFmpeg-Builds' outputs: - random-number: - description: "Random number" - value: ${{ steps.first.outputs.random-number }} + releases: + value: ${{ steps.set.outputs.releases }} + description: 'Generated JSON describing the released builds.' runs: using: 'composite' steps: - - name: First step - id: 'first' + - name: Retrieve releases + id: 'set' env: - INPUT_WHO_TO_GREET: ${{ inputs.who-to-greet }} + GH_TOKEN: '${{ inputs.token }}' + GH_API_GQL_ASSETS: '${{ inputs.num-assets }}' + GH_API_GQL_RELEASES: '${{ inputs.num-releases }}' + GH_API_GQL_OWNER: '${{ inputs.repository_owner }}' + GH_API_GQL_REPO: '${{ inputs.repository_name }}' shell: 'bash' run: | - echo 'It worked!' - echo "random-number=${RANDOM}" >> "${GITHUB_OUTPUT}" - ls -al '${{ github.action_path }}' - echo "Hello ${INPUT_WHO_TO_GREET}." + command -v gh + command -v jq + gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; + { + var='releases' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" ; + close_ml_var "${delim}" "${var}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" ; + # Log the human version + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; From 41f37bcd656c5c075f8ffbb6b7d1b130c745f134 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:00:46 -0400 Subject: [PATCH 23/28] Use the new `FFmpeg` action --- .github/workflows/ci.yaml | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e395239c..2b465feb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ on: jobs: info: - if: ${{ !cancelled() && 'pull_request' != github.event_name }} + #if: ${{ !cancelled() && 'pull_request' != github.event_name }} runs-on: ubuntu-latest outputs: ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} @@ -49,40 +49,10 @@ jobs: "${k}" "${v,,}" done unset -v k v var + - uses: actions/checkout@v4 - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI id: ffmpeg - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_API_GQL_ASSETS: 25 - GH_API_GQL_RELEASES: 35 - GH_API_GQL_OWNER: yt-dlp - GH_API_GQL_REPO: FFmpeg-Builds - run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; - gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; - mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; - open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; - close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; - { - var='releases' ; - delim="$(mk_delim "${var}")" ; - open_ml_var "${delim}" "${var}" ; - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" ; - close_ml_var "${delim}" "${var}" ; - unset -v delim jq_arg var ; - } >> "${GITHUB_OUTPUT}" ; - # Log the human version - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; + uses: ./.github/actions/FFmpeg - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI id: yt-dlp env: From b30d0ce88f281a9fc41c182bc4e1d6b0eb10c275 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:07:17 -0400 Subject: [PATCH 24/28] Debug the shell code --- .github/workflows/ci.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b465feb..063b50ee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,17 +38,18 @@ jobs: id: lowercase-variables shell: bash run: | + set -x ; set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ; for var in \ actor='${{ github.actor }}' \ repository_owner='${{ github.repository_owner }}' do - k="$(cut -d '=' -f 1)" - v="${var#${k}=}" + k="$(cut -d '=' -f 1)" ; + v="${var#${k}=}" ; set_sl_var >> "${GITHUB_OUTPUT}" \ - "${k}" "${v,,}" - done - unset -v k v var + "${k}" "${v,,}" ; + done ; + unset -v k v var ; - uses: actions/checkout@v4 - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI id: ffmpeg From d556f203012cd7b2528363aad9cefede91929a04 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:10:21 -0400 Subject: [PATCH 25/28] fixup: cut needed some input --- .github/workflows/ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 063b50ee..2b70c334 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,13 +38,12 @@ jobs: id: lowercase-variables shell: bash run: | - set -x ; set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ; - for var in \ + set -x ; for var in \ actor='${{ github.actor }}' \ repository_owner='${{ github.repository_owner }}' do - k="$(cut -d '=' -f 1)" ; + k="$( cut -d '=' -f 1 <<<"${var}" )" ; v="${var#${k}=}" ; set_sl_var >> "${GITHUB_OUTPUT}" \ "${k}" "${v,,}" ; From f4e9c08376f762e2089e273f4bda51794c1a9708 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:19:47 -0400 Subject: [PATCH 26/28] Use `github.token` instead of `secrets.GITHUB_TOKEN` --- .github/actions/FFmpeg/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml index 90a1210f..4e20ddc0 100644 --- a/.github/actions/FFmpeg/action.yml +++ b/.github/actions/FFmpeg/action.yml @@ -4,10 +4,10 @@ description: 'Use GitHub CLI & API to retrieve information about FFmpeg Build re inputs: token: required: true - default: ${{ secrets.GITHUB_TOKEN }} + default: ${{ github.token }} description: | GH_TOKEN for GitHub CLI to use. - Default: $${{ secrets.GITHUB_TOKEN }} + Default: $${{ github.token }} num-assets: required: true default: '25' @@ -44,7 +44,7 @@ runs: - name: Retrieve releases id: 'set' env: - GH_TOKEN: '${{ inputs.token }}' + GH_TOKEN: ${{ inputs.token }} GH_API_GQL_ASSETS: '${{ inputs.num-assets }}' GH_API_GQL_RELEASES: '${{ inputs.num-releases }}' GH_API_GQL_OWNER: '${{ inputs.repository_owner }}' From 97e69e415645a2943a596225a121b4f6fc5ba7a2 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:29:29 -0400 Subject: [PATCH 27/28] Try without the braces --- .github/actions/FFmpeg/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml index 4e20ddc0..db9b61bd 100644 --- a/.github/actions/FFmpeg/action.yml +++ b/.github/actions/FFmpeg/action.yml @@ -7,7 +7,7 @@ inputs: default: ${{ github.token }} description: | GH_TOKEN for GitHub CLI to use. - Default: $${{ github.token }} + Default: github.token num-assets: required: true default: '25' From 4e82fcb5ccd0559c08e92223a13470c5fc4e1acf Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 26 Mar 2025 22:42:02 -0400 Subject: [PATCH 28/28] fixup: escaping the sequence --- .github/actions/FFmpeg/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml index db9b61bd..40170ccb 100644 --- a/.github/actions/FFmpeg/action.yml +++ b/.github/actions/FFmpeg/action.yml @@ -7,7 +7,7 @@ inputs: default: ${{ github.token }} description: | GH_TOKEN for GitHub CLI to use. - Default: github.token + Default: `\$\{\{ github.token \}\}` num-assets: required: true default: '25'