diff --git a/.github/actions/FFmpeg/action.yml b/.github/actions/FFmpeg/action.yml new file mode 100644 index 00000000..40170ccb --- /dev/null +++ b/.github/actions/FFmpeg/action.yml @@ -0,0 +1,80 @@ +name: 'FFmpeg Builds' +description: 'Use GitHub CLI & API to retrieve information about FFmpeg Build releases.' + +inputs: + token: + required: true + default: ${{ github.token }} + description: | + GH_TOKEN for GitHub CLI to use. + Default: `\$\{\{ github.token \}\}` + num-assets: + required: true + default: '25' + description: | + The number of assets (attached files) to retrieve from each release. + Default: 25 + num-releases: + required: true + default: '35' + description: | + The number of releases to retrieve from the repository. + Default: 35 + repository_owner: + required: true + default: 'yt-dlp' + description: | + The name of the user or organization that owns the repository. + Default: 'yt-dlp' + repository_name: + required: true + default: 'FFmpeg-Builds' + description: | + Which repository from the owner to search for releases. + Default: 'FFmpeg-Builds' + +outputs: + releases: + value: ${{ steps.set.outputs.releases }} + description: 'Generated JSON describing the released builds.' + +runs: + using: 'composite' + steps: + - name: Retrieve releases + id: 'set' + env: + GH_TOKEN: ${{ inputs.token }} + GH_API_GQL_ASSETS: '${{ inputs.num-assets }}' + GH_API_GQL_RELEASES: '${{ inputs.num-releases }}' + GH_API_GQL_OWNER: '${{ inputs.repository_owner }}' + GH_API_GQL_REPO: '${{ inputs.repository_name }}' + shell: 'bash' + run: | + command -v gh + command -v jq + gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; + { + var='releases' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" ; + close_ml_var "${delim}" "${var}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" ; + # Log the human version + gh api graphql --cache 12h \ + -F assets="${GH_API_GQL_ASSETS}" \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7eef6ea..2b70c334 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Run Django tests for TubeSync +name: CI env: IMAGE_NAME: tubesync @@ -8,9 +8,95 @@ on: push: branches: - main + pull_request: + branches: + - main + types: + - opened + - reopened + - synchronize + - ready_for_review jobs: + info: + #if: ${{ !cancelled() && 'pull_request' != github.event_name }} + runs-on: ubuntu-latest + outputs: + ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }} + lowercase-variables-actor: ${{ steps.lowercase-variables.outputs.actor }} + lowercase-variables-repository_owner: ${{ steps.lowercase-variables.outputs.repository_owner }} + string-lowercase: ${{ steps.string.outputs.lowercase }} + ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }} + ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }} + steps: + - name: Lowercase github username for ghcr + id: string + uses: ASzc/change-string-case-action@v6 + with: + string: ${{ github.actor }} + - name: Lowercase GitHub variables + id: lowercase-variables + shell: bash + run: | + set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ; + set -x ; for var in \ + actor='${{ github.actor }}' \ + repository_owner='${{ github.repository_owner }}' + do + k="$( cut -d '=' -f 1 <<<"${var}" )" ; + v="${var#${k}=}" ; + set_sl_var >> "${GITHUB_OUTPUT}" \ + "${k}" "${v,,}" ; + done ; + unset -v k v var ; + - uses: actions/checkout@v4 + - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI + id: ffmpeg + uses: ./.github/actions/FFmpeg + - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI + id: yt-dlp + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_API_GQL_RELEASES: 25 + GH_API_GQL_OWNER: yt-dlp + GH_API_GQL_REPO: yt-dlp + run: | + gql_query='query($repo: String!, $owner: String!, $releases: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC }) { nodes { name, createdAt, publishedAt, updatedAt, tagName, url, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } } } } } }' ; + gql_jq='[ .data.repository.releases.nodes[] | select((.isDraft or .isPrerelease) | not) | del(.isDraft, .isPrerelease) ]' ; + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; + { + var='releases' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" ; + close_ml_var "${delim}" "${var}" ; + jq_arg='map(select(.isLatest))[0]' ; + var='latest-release' ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq -c "${jq_arg}" -- ; + close_ml_var "${delim}" "${var}" ; + unset -v delim jq_arg var ; + } >> "${GITHUB_OUTPUT}" ; + # Log the human version + gh api graphql --cache 12h \ + -F owner="${GH_API_GQL_OWNER}" \ + -F repo="${GH_API_GQL_REPO}" \ + -F releases="${GH_API_GQL_RELEASES}" \ + -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; + test: + if: ${{ !cancelled() && ( 'pull_request' != github.event_name || (! github.event.pull_request.draft) ) }} runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -34,63 +120,37 @@ jobs: cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* - name: Run Django tests run: cd tubesync && python3 manage.py test --verbosity=2 + containerise: - if: ${{ !cancelled() }} - needs: test + if: ${{ !cancelled() && 'success' == needs.info.result }} + needs: ['info', 'test'] runs-on: ubuntu-latest timeout-minutes: 120 steps: - - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI - id: ffmpeg - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_API_GQL_ASSETS: 25 - GH_API_GQL_RELEASES: 35 - GH_API_GQL_OWNER: yt-dlp - GH_API_GQL_REPO: FFmpeg-Builds - run: | - gql_query='query($repo: String!, $owner: String!, $releases: Int!, $assets: Int!) { repository(owner: $owner, name: $repo) { releases(first: $releases, orderBy: { field: CREATED_AT, direction: DESC}) { nodes { tagName, isDraft, isPrerelease, isLatest, tag { name, target { oid, commitUrl } }, releaseAssets(first: $assets) { totalCount, nodes { name, size, downloadUrl } } } } } }' ; - gql_jq='[ .data.repository.releases.nodes[] | select((.isLatest or .isDraft or .isPrerelease) | not) | { "tag": .tag.name, "commit": .tag.target.oid, "date": .tag.name[1+(.tag.name|index("-")):], "assets": { "limit": '"${GH_API_GQL_ASSETS}"', "totalCount": .releaseAssets.totalCount }, "files": .releaseAssets.nodes, "versions": [ .releaseAssets.nodes[].name | select(contains("-linux64-"))[1+index("-"):index("-linux64-")] ] } ]' ; - { - var='releases' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" ; - printf -- '%s\n' "${delim}" ; - unset -v delim jq_arg var ; - } >> "${GITHUB_OUTPUT}" - gh api graphql --cache 12h \ - -F assets="${GH_API_GQL_ASSETS}" \ - -F owner="${GH_API_GQL_OWNER}" \ - -F repo="${GH_API_GQL_REPO}" \ - -F releases="${GH_API_GQL_RELEASES}" \ - -f query="${gql_query}" --jq "${gql_jq}" | jq '.[]' -- ; - name: Set environment variables with jq run: | cat >| .ffmpeg.releases.json <<'EOF' - ${{ steps.ffmpeg.outputs.releases }} + ${{ needs.info.outputs.ffmpeg-releases }} EOF + mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ; + open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ; + close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ; { var='FFMPEG_DATE' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ; jq -r "${jq_arg}" -- .ffmpeg.releases.json ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )" var='FFMPEG_VERSION' ; - delim='"'"${var}"'_EOF"' ; - printf -- '%s<<%s\n' "${var}" "${delim}" ; + delim="$(mk_delim "${var}")" ; + open_ml_var "${delim}" "${var}" ; jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ; jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ; - printf -- '%s\n' "${delim}" ; + close_ml_var "${delim}" "${var}" ; unset -v delim jq_arg var ; } >> "${GITHUB_ENV}" - name: Set up QEMU @@ -104,21 +164,22 @@ jobs: DOCKER_USERNAME: ${{ github.actor }} DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}" - - name: Lowercase github username for ghcr - id: string - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.actor }} - name: Build and push timeout-minutes: 60 uses: docker/build-push-action@v6 with: platforms: linux/amd64,linux/arm64 - push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'true' || 'false' }} - tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-to: type=inline + push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }} + tags: ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest + cache-from: | + type=registry,ref=ghcr.io/${{ needs.info.outputs.string-lowercase }}/${{ env.IMAGE_NAME }}:latest + type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest + type=gha + cache-to: | + type=gha,mode=max + ${{ 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'type=inline' || '' }} build-args: | IMAGE_NAME=${{ env.IMAGE_NAME }} FFMPEG_DATE=${{ env.FFMPEG_DATE }} FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} + YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }} diff --git a/Dockerfile b/Dockerfile index c0fcd9ad..82bc665e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -315,6 +315,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va # Switch workdir to the the app WORKDIR /app +ARG YTDLP_DATE + # Set up the app RUN --mount=type=tmpfs,target=/cache \ --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \