Merge pull request #619 from tcely/patch-6

Multi-stage docker build for ffmpeg & s6-overlay

Dockerfile

@@ -1,17 +1,231 @@
-FROM debian:bookworm-slim
+# syntax=docker/dockerfile:1
+# check=error=true
 
-ARG TARGETARCH
+ARG FFMPEG_DATE="2025-01-10-19-43"
-ARG TARGETPLATFORM
+ARG FFMPEG_VERSION="N-118280-g5cd49e1bfd"
 
 ARG S6_VERSION="3.2.0.2"
+
 ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478"
 ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785"
 ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae"
 
-ARG FFMPEG_DATE="autobuild-2024-12-24-14-15"
+ARG ALPINE_VERSION="latest"
-ARG FFMPEG_VERSION="N-118163-g954d55c2a4"
+ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION}"
-ARG SHA256_FFMPEG_AMD64="798a7e5a0724139e6bb70df8921522b23be27028f9f551dfa83c305ec4ffaf3a"
+ARG FFMPEG_SUFFIX_FILE=".tar.xz"
-ARG SHA256_FFMPEG_ARM64="c3e6cc0fec42cc7e3804014fbb02c1384a1a31ef13f6f9a36121f2e1216240c0"
+
+ARG FFMPEG_CHECKSUM_ALGORITHM="sha256"
+ARG S6_CHECKSUM_ALGORITHM="sha256"
+
+FROM alpine:${ALPINE_VERSION} AS ffmpeg-download
+ARG FFMPEG_DATE
+ARG FFMPEG_VERSION
+ARG FFMPEG_PREFIX_FILE
+ARG FFMPEG_SUFFIX_FILE
+ARG SHA256_FFMPEG_AMD64
+ARG SHA256_FFMPEG_ARM64
+ARG FFMPEG_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${FFMPEG_CHECKSUM_ALGORITHM}"
+ARG FFMPEG_CHECKSUM_AMD64="${SHA256_FFMPEG_AMD64}"
+ARG FFMPEG_CHECKSUM_ARM64="${SHA256_FFMPEG_ARM64}"
+
+ARG FFMPEG_FILE_SUMS="checksums.${CHECKSUM_ALGORITHM}"
+ARG FFMPEG_URL="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-${FFMPEG_DATE}"
+
+ARG DESTDIR="/downloaded"
+ARG TARGETARCH
+ADD "${FFMPEG_URL}/${FFMPEG_FILE_SUMS}" "${DESTDIR}/"
+RUN set -eu ; \
+    apk --no-cache --no-progress add cmd:aria2c cmd:awk ; \
+\
+    aria2c_options() { \
+        algorithm="${CHECKSUM_ALGORITHM%[0-9]??}" ; \
+        bytes="${CHECKSUM_ALGORITHM#${algorithm}}" ; \
+        hash="$( awk -v fn="${1##*/}" '$0 ~ fn"$" { print $1; exit; }' "${DESTDIR}/${FFMPEG_FILE_SUMS}" )" ; \
+\
+        printf -- '\t%s\n' \
+          'allow-overwrite=true' \
+          'always-resume=false' \
+          'check-integrity=true' \
+          "checksum=${algorithm}-${bytes}=${hash}" \
+          'max-connection-per-server=2' \
+; \
+        printf -- '\n' ; \
+    } ; \
+\
+    decide_arch() { \
+        case "${TARGETARCH}" in \
+            (amd64) printf -- 'linux64' ;; \
+            (arm64) printf -- 'linuxarm64' ;; \
+        esac ; \
+    } ; \
+\
+    FFMPEG_ARCH="$(decide_arch)" ; \
+    FFMPEG_PREFIX_FILE="$( printf -- '%s' "${FFMPEG_PREFIX_FILE}" | cut -d '-' -f 1,2 )" ; \
+    for url in $(awk ' \
+      $2 ~ /^[*]?'"${FFMPEG_PREFIX_FILE}"'/ && /-'"${FFMPEG_ARCH}"'-/ { $1=""; print; } \
+      ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \
+    do \
+        url="${FFMPEG_URL}/${url# }" ; \
+        printf -- '%s\n' "${url}" ; \
+        aria2c_options "${url}" ; \
+        printf -- '\n' ; \
+    done > /tmp/downloads ; \
+    unset -v url ; \
+\
+    aria2c --no-conf=true \
+      --dir /downloaded \
+      --lowest-speed-limit='16K' \
+      --show-console-readout=false \
+      --summary-interval=0 \
+      --input-file /tmp/downloads ; \
+\
+    apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \
+\    
+    decide_expected() { \
+        case "${TARGETARCH}" in \
+            (amd64) printf -- '%s' "${FFMPEG_CHECKSUM_AMD64}" ;; \
+            (arm64) printf -- '%s' "${FFMPEG_CHECKSUM_ARM64}" ;; \
+        esac ; \
+    } ; \
+\
+    FFMPEG_HASH="$(decide_expected)" ; \
+\
+    cd "${DESTDIR}" ; \
+    if [ -n "${FFMPEG_HASH}" ] ; \
+    then \
+        printf -- '%s *%s\n' "${FFMPEG_HASH}" "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" >> /tmp/SUMS ; \
+        "${CHECKSUM_ALGORITHM}sum" --check --warn --strict /tmp/SUMS || exit ; \
+    fi ; \
+    "${CHECKSUM_ALGORITHM}sum" --check --warn --strict --ignore-missing "${DESTDIR}/${FFMPEG_FILE_SUMS}" ; \
+\
+    mkdir -v -p "/verified/${TARGETARCH}" ; \
+    ln -v "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" "/verified/${TARGETARCH}/" ; \
+    rm -rf "${DESTDIR}" ;
+
+FROM alpine:${ALPINE_VERSION} AS ffmpeg-extracted
+COPY --from=ffmpeg-download /verified /verified
+
+ARG FFMPEG_PREFIX_FILE
+ARG FFMPEG_SUFFIX_FILE
+ARG TARGETARCH
+RUN set -eux ; \
+    mkdir -v /extracted ; \
+    cd /extracted ; \
+    ln -s "/verified/${TARGETARCH}"/"${FFMPEG_PREFIX_FILE}"*"${FFMPEG_SUFFIX_FILE}" "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" ; \
+    tar -tf "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" | grep '/bin/\(ffmpeg\|ffprobe\)' > /tmp/files ; \
+    tar -xop \
+      --strip-components=2 \
+      -f "/tmp/ffmpeg${FFMPEG_SUFFIX_FILE}" \
+      -T /tmp/files ; \
+\
+    ls -AlR /extracted ;
+
+FROM scratch AS ffmpeg
+COPY --from=ffmpeg-extracted /extracted /usr/local/bin/
+
+FROM alpine:${ALPINE_VERSION} AS s6-overlay-download
+ARG S6_VERSION
+ARG SHA256_S6_AMD64
+ARG SHA256_S6_ARM64
+ARG SHA256_S6_NOARCH
+
+ARG DESTDIR="/downloaded"
+ARG S6_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${S6_CHECKSUM_ALGORITHM}"
+
+ARG S6_CHECKSUM_AMD64="${CHECKSUM_ALGORITHM}:${SHA256_S6_AMD64}"
+ARG S6_CHECKSUM_ARM64="${CHECKSUM_ALGORITHM}:${SHA256_S6_ARM64}"
+ARG S6_CHECKSUM_NOARCH="${CHECKSUM_ALGORITHM}:${SHA256_S6_NOARCH}"
+
+ARG S6_OVERLAY_URL="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}"
+ARG S6_PREFIX_FILE="s6-overlay-"
+ARG S6_SUFFIX_FILE=".tar.xz"
+
+ARG S6_FILE_AMD64="${S6_PREFIX_FILE}x86_64${S6_SUFFIX_FILE}"
+ARG S6_FILE_ARM64="${S6_PREFIX_FILE}aarch64${S6_SUFFIX_FILE}"
+ARG S6_FILE_NOARCH="${S6_PREFIX_FILE}noarch${S6_SUFFIX_FILE}"
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/"
+
+##ADD --checksum="${S6_CHECKSUM_AMD64}" "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
+##ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
+##ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+
+# --checksum wasn't recognized, so use busybox to check the sums instead
+ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_AMD64}" ; file="${S6_FILE_AMD64}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_ARM64}" ; file="${S6_FILE_ARM64}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
+
+ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/"
+RUN set -eu ; checksum="${S6_CHECKSUM_NOARCH}" ; file="${S6_FILE_NOARCH}" ; cd "${DESTDIR}/" && \
+    printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw
+
+FROM alpine:${ALPINE_VERSION} AS s6-overlay-extracted
+COPY --from=s6-overlay-download /downloaded /downloaded
+
+ARG S6_CHECKSUM_ALGORITHM
+ARG CHECKSUM_ALGORITHM="${S6_CHECKSUM_ALGORITHM}"
+
+ARG TARGETARCH
+
+RUN set -eu ; \
+\
+    decide_arch() { \
+      local arg1 ; \
+      arg1="${1:-$(uname -m)}" ; \
+\
+      case "${arg1}" in \
+        (amd64) printf -- 'x86_64' ;; \
+        (arm64) printf -- 'aarch64' ;; \
+        (armv7l) printf -- 'arm' ;; \
+        (*) printf -- '%s' "${arg1}" ;; \
+      esac ; \
+      unset -v arg1 ; \
+    } ; \
+\
+    apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \
+    mkdir -v /verified ; \
+    cd /downloaded ; \
+    for f in *.sha256 ; \
+    do \
+      "${CHECKSUM_ALGORITHM}sum" --check --warn --strict "${f}" || exit ; \
+      ln -v "${f%.sha256}" /verified/ || exit ; \
+    done ; \
+    unset -v f ; \
+\
+    S6_ARCH="$(decide_arch "${TARGETARCH}")" ; \
+    set -x ; \
+    mkdir -v /s6-overlay-rootfs ; \
+    cd /s6-overlay-rootfs ; \
+    for f in /verified/*.tar* ; \
+    do \
+      case "${f}" in \
+        (*-noarch.tar*|*-"${S6_ARCH}".tar*) \
+          tar -xpf "${f}" || exit ;; \
+      esac ; \
+    done ; \
+    set +x ; \
+    unset -v f ;
+
+FROM scratch AS s6-overlay
+COPY --from=s6-overlay-extracted /s6-overlay-rootfs /
+
+FROM debian:bookworm-slim AS tubesync
+
+ARG TARGETARCH
+ARG TARGETPLATFORM
+
+ARG S6_VERSION
+
+ARG FFMPEG_DATE
+ARG FFMPEG_VERSION
 
 ENV S6_VERSION="${S6_VERSION}" \
   FFMPEG_DATE="${FFMPEG_DATE}" \
@@ -26,89 +240,20 @@ ENV DEBIAN_FRONTEND="noninteractive" \
   S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
 
 # Install third party software
+COPY --from=s6-overlay / /
+COPY --from=ffmpeg /usr/local/bin/ /usr/local/bin/
+
 # Reminder: the SHELL handles all variables
-RUN decide_arch() { \
+RUN set -x && \
-      case "${TARGETARCH:=amd64}" in \
-        (arm64) printf -- 'aarch64' ;; \
-        (*) printf -- '%s' "${TARGETARCH}" ;; \
-      esac ; \
-    } && \
-    decide_expected() { \
-      case "${1}" in \
-        (ffmpeg) case "${2}" in \
-            (amd64) printf -- '%s' "${SHA256_FFMPEG_AMD64}" ;; \
-            (arm64) printf -- '%s' "${SHA256_FFMPEG_ARM64}" ;; \
-          esac ;; \
-        (s6) case "${2}" in \
-            (amd64) printf -- '%s' "${SHA256_S6_AMD64}" ;; \
-            (arm64) printf -- '%s' "${SHA256_S6_ARM64}" ;; \
-            (noarch) printf -- '%s' "${SHA256_S6_NOARCH}" ;; \
-          esac ;; \
-      esac ; \
-    } && \
-    decide_url() { \
-      case "${1}" in \
-        (ffmpeg) printf -- \
-          'https://github.com/yt-dlp/FFmpeg-Builds/releases/download/%s/ffmpeg-%s-linux%s-gpl%s.tar.xz' \
-          "${FFMPEG_DATE}" \
-          "${FFMPEG_VERSION}" \
-          "$(case "${2}" in \
-            (amd64) printf -- '64' ;; \
-            (*) printf -- '%s' "${2}" ;; \
-          esac)" \
-          "$(case "${FFMPEG_VERSION%%-*}" in \
-            (n*) printf -- '-%s\n' "${FFMPEG_VERSION#n}" | cut -d '-' -f 1,2 ;; \
-            (*) printf -- '' ;; \
-          esac)" ;; \
-        (s6) printf -- \
-          'https://github.com/just-containers/s6-overlay/releases/download/v%s/s6-overlay-%s.tar.xz' \
-          "${S6_VERSION}" \
-          "$(case "${2}" in \
-            (amd64) printf -- 'x86_64' ;; \
-            (arm64) printf -- 'aarch64' ;; \
-            (*) printf -- '%s' "${2}" ;; \
-          esac)" ;; \
-      esac ; \
-    } && \
-    verify_download() { \
-      while [ $# -ge 2 ] ; do \
-        sha256sum "${2}" ; \
-        printf -- '%s  %s\n' "${1}" "${2}" | sha256sum -c || return ; \
-        shift ; shift ; \
-      done ; \
-    } && \
-    download_expected_file() { \
-      local arg1 expected file url ; \
-      arg1="$(printf -- '%s\n' "${1}" | awk '{print toupper($0);}')" ; \
-      expected="$(decide_expected "${1}" "${2}")" ; \
-      file="${3}" ; \
-      url="$(decide_url "${1}" "${2}")" ; \
-      printf -- '%s\n' \
-        "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \
-      rm -rf "${file}" && \
-      curl --disable --output "${file}" --clobber --location --no-progress-meter --url "${url}" && \
-      verify_download "${expected}" "${file}" ; \
-    } && \
-  export ARCH="$(decide_arch)" && \
-  set -x && \
   apt-get update && \
   apt-get -y --no-install-recommends install locales && \
   printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \
   locale-gen en_US.UTF-8 && \
   # Install required distro packages
   apt-get -y --no-install-recommends install curl ca-certificates file binutils xz-utils && \
-  # Install s6
+  # Installed s6 (using COPY earlier)
-  _file="/tmp/s6-overlay-noarch.tar.xz" && \
-  download_expected_file s6 noarch "${_file}" && \
-  tar -C / -xpf "${_file}" && rm -f "${_file}" && \
-  _file="/tmp/s6-overlay-${ARCH}.tar.xz" && \
-  download_expected_file s6 "${TARGETARCH}" "${_file}" && \
-  tar -C / -xpf "${_file}" && rm -f "${_file}" && \
   file -L /command/s6-overlay-suexec && \
-  # Install ffmpeg
+  # Installed ffmpeg (using COPY earlier)
-  _file="/tmp/ffmpeg-${ARCH}.tar.xz" && \
-  download_expected_file ffmpeg "${TARGETARCH}" "${_file}" && \
-  tar -xvvpf "${_file}" --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" "ffprobe" && rm -f "${_file}" && \
   /usr/local/bin/ffmpeg -version && \
   file /usr/local/bin/ff* && \
   # Clean up
@@ -154,7 +299,9 @@ ENV PIP_NO_COMPILE=1 \
 WORKDIR /app
 
 # Set up the app
-RUN set -x && \
+#BuildKit#RUN --mount=type=bind,source=Pipfile,target=/app/Pipfile \
+RUN \
+  set -x && \
   apt-get update && \
   # Install required build packages
   apt-get -y --no-install-recommends install \