From a60f68faf3b9fa82f4ffae3d2e6b7b460255da1d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 00:15:28 -0500 Subject: [PATCH 01/14] Used printf consistently Specifically turn off flags to printf where none are intended. There are fewer things to keep in mind this way. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 32314ed6..1c7751f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,8 @@ ENV DEBIAN_FRONTEND="noninteractive" \ # Reminder: the SHELL handles all variables RUN decide_arch() { \ case "${TARGETARCH:=amd64}" in \ - (arm64) printf 'aarch64' ;; \ - (*) printf '%s' "${TARGETARCH}" ;; \ + (arm64) printf -- 'aarch64' ;; \ + (*) printf -- '%s' "${TARGETARCH}" ;; \ esac ; \ } && \ decide_expected() { \ @@ -88,7 +88,7 @@ RUN decide_arch() { \ set -x && \ apt-get update && \ apt-get -y --no-install-recommends install locales && \ - echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \ locale-gen en_US.UTF-8 && \ # Install required distro packages apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \ @@ -205,7 +205,7 @@ RUN set -x && \ # Append software versions RUN set -x && \ FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \ - echo "ffmpeg_version = '${FFMPEG_VERSION}'" >> /app/common/third_party_versions.py + printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py # Copy root COPY config/root / From 401c687725fb5b296a0bbe760028ab78c3a68074 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 00:18:51 -0500 Subject: [PATCH 02/14] Adjusted HOME only for pipenv Nothing later should have been affected, but this is the cleaner way. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1c7751f9..96cd312e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,8 +159,8 @@ RUN set -x && \ groupadd app && \ useradd -M -d /app -s /bin/false -g app app && \ # Install non-distro packages - cp -at /tmp/ "${HOME}" && HOME="/tmp/${HOME#/}" && \ - PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \ + cp -at /tmp/ "${HOME}" && \ + PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock && \ # Clean up rm /app/Pipfile && \ pipenv --clear && \ From 908b55c191224616b176bd7b2daea84d3e5d6f2c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 00:21:57 -0500 Subject: [PATCH 03/14] Show which directories were created --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96cd312e..60c8937b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -196,10 +196,10 @@ RUN set -x && \ /usr/bin/python3 /app/manage.py compilescss && \ /usr/bin/python3 /app/manage.py collectstatic --no-input --link && \ # Create config, downloads and run dirs - mkdir -p /run/app && \ - mkdir -p /config/media && \ - mkdir -p /downloads/audio && \ - mkdir -p /downloads/video + mkdir -v -p /run/app && \ + mkdir -v -p /config/media && \ + mkdir -v -p /downloads/audio && \ + mkdir -v -p /downloads/video # Append software versions From d4834e427f6f22c2cb0cfd0cff1ac704673f1d37 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 00:27:33 -0500 Subject: [PATCH 04/14] Failed ffmpeg is now noisier This should be enough to fail the build when the binary doesn't work. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 60c8937b..c513ee61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -204,7 +204,8 @@ RUN set -x && \ # Append software versions RUN set -x && \ - FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \ + /usr/local/bin/ffmpeg -version && \ + FFMPEG_VERSION=$(set -o pipefail ; /usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }' || exit) && \ printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py # Copy root From 10678afca24a325089c0c60ed269347287395171 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 00:33:45 -0500 Subject: [PATCH 05/14] Log what file determined about the extracted files --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c513ee61..06c7ac33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,7 +91,7 @@ RUN decide_arch() { \ printf -- "en_US.UTF-8 UTF-8\n" > /etc/locale.gen && \ locale-gen en_US.UTF-8 && \ # Install required distro packages - apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \ + apt-get -y --no-install-recommends install curl ca-certificates file binutils xz-utils && \ # Install s6 _file="/tmp/s6-overlay-noarch.tar.xz" && \ download_expected_file s6 noarch "${_file}" && \ @@ -99,12 +99,14 @@ RUN decide_arch() { \ _file="/tmp/s6-overlay-${ARCH}.tar.xz" && \ download_expected_file s6 "${TARGETARCH}" "${_file}" && \ tar -C / -xpf "${_file}" && rm -f "${_file}" && \ + file /init && \ # Install ffmpeg _file="/tmp/ffmpeg-${ARCH}.tar.xz" && \ download_expected_file ffmpeg "${TARGETARCH}" "${_file}" && \ tar -xvvpf "${_file}" --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" "ffprobe" && rm -f "${_file}" && \ + file /usr/local/bin/ff* && \ # Clean up - apt-get -y autoremove --purge curl binutils xz-utils && \ + apt-get -y autoremove --purge curl file binutils xz-utils && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /var/cache/apt/* && \ rm -rf /tmp/* From 599728d38c1be0e12180816cdabee8ac2ed88de0 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 05:33:53 -0500 Subject: [PATCH 06/14] Had file inspect the binary from the /init script instead. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 06c7ac33..e961b9e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,7 @@ RUN decide_arch() { \ _file="/tmp/s6-overlay-${ARCH}.tar.xz" && \ download_expected_file s6 "${TARGETARCH}" "${_file}" && \ tar -C / -xpf "${_file}" && rm -f "${_file}" && \ - file /init && \ + file -L /command/s6-overlay-suexec && \ # Install ffmpeg _file="/tmp/ffmpeg-${ARCH}.tar.xz" && \ download_expected_file ffmpeg "${TARGETARCH}" "${_file}" && \ From 3c5641b542e8bc8c399f8b596730542e086b57fa Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 05:44:22 -0500 Subject: [PATCH 07/14] Ignore a curl config file that may exist --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e961b9e8..b56efa59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,7 @@ RUN decide_arch() { \ url="$(decide_url "${1}" "${2}")" ; \ printf -- '%s\n' \ "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \ - curl -sSL --output "${file}" "${url}" && \ + curl --disable -sSL --output "${file}" "${url}" && \ verify_download "${expected}" "${file}" ; \ } && \ export ARCH="$(decide_arch)" && \ From 6b807a8654eea0094a1a04a67eb45106c321e9af Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 05:47:40 -0500 Subject: [PATCH 08/14] Log the server response headers --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b56efa59..16d80c09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,7 @@ RUN decide_arch() { \ url="$(decide_url "${1}" "${2}")" ; \ printf -- '%s\n' \ "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \ - curl --disable -sSL --output "${file}" "${url}" && \ + curl --disable -sSL --output "${file}" --dump-header - "${url}" && \ verify_download "${expected}" "${file}" ; \ } && \ export ARCH="$(decide_arch)" && \ From 163fcbeb2a09d708f7cb2c67bb97c0d44134b16e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 06:10:30 -0500 Subject: [PATCH 09/14] Overwrite the specified file when it already exists --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 16d80c09..606984ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,7 +81,8 @@ RUN decide_arch() { \ url="$(decide_url "${1}" "${2}")" ; \ printf -- '%s\n' \ "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \ - curl --disable -sSL --output "${file}" --dump-header - "${url}" && \ + rm -rf "${file}" && \ + curl --disable -SL --output "${file}" --clobber --dump-header - --no-progress-meter "${url}" && \ verify_download "${expected}" "${file}" ; \ } && \ export ARCH="$(decide_arch)" && \ From f4a88cf7fa4d78061eabd12df927fcf4518a2c35 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 06:34:28 -0500 Subject: [PATCH 10/14] Used the long form for all curl options. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 606984ea..20466e95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ RUN decide_arch() { \ printf -- '%s\n' \ "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \ rm -rf "${file}" && \ - curl --disable -SL --output "${file}" --clobber --dump-header - --no-progress-meter "${url}" && \ + curl --disable --output "${file}" --clobber --location --dump-header - --no-progress-meter --url "${url}" && \ verify_download "${expected}" "${file}" ; \ } && \ export ARCH="$(decide_arch)" && \ From a921adedc0d333c0431f8222e1bc6aab4dfaf8af Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 07:13:19 -0500 Subject: [PATCH 11/14] Test the output for FFMPEG_VERSION exists The shell used on the build system doesn't support `pipefail` so test for output instead. It might be incorrect output, but in combination with running the `ffmpeg` binary earlier, this should be enough. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 20466e95..130b54cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -208,7 +208,8 @@ RUN set -x && \ # Append software versions RUN set -x && \ /usr/local/bin/ffmpeg -version && \ - FFMPEG_VERSION=$(set -o pipefail ; /usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }' || exit) && \ + FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \ + test -n "${FFMPEG_VERSION}" && \ printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py # Copy root From 29dfe4570d463545a19ea1a6ec3e4a1ae75de287 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 08:16:34 -0500 Subject: [PATCH 12/14] Used awk to check for correct ffmpeg output --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 130b54cc..9bc3272e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -208,7 +208,7 @@ RUN set -x && \ # Append software versions RUN set -x && \ /usr/local/bin/ffmpeg -version && \ - FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \ + FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | awk '1 == NR && "ffmpeg" == $1 { print $3; exit 0; } END { exit 1; }') && \ test -n "${FFMPEG_VERSION}" && \ printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py From 6c7c1c6510a55806d2a5a6e9e00893a7c7c18c1a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 24 Nov 2024 08:34:34 -0500 Subject: [PATCH 13/14] The response headers weren't useful --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9bc3272e..2d688e0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ RUN decide_arch() { \ printf -- '%s\n' \ "Building for arch: ${2}|${ARCH}, downloading ${arg1} from: ${url}, expecting ${arg1} SHA256: ${expected}" && \ rm -rf "${file}" && \ - curl --disable --output "${file}" --clobber --location --dump-header - --no-progress-meter --url "${url}" && \ + curl --disable --output "${file}" --clobber --location --no-progress-meter --url "${url}" && \ verify_download "${expected}" "${file}" ; \ } && \ export ARCH="$(decide_arch)" && \ From 8f9bb782a775b820396f182115d7db7921caafb8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 25 Nov 2024 17:00:31 -0500 Subject: [PATCH 14/14] Adjusted awk check It turned out `exit` doesn't work as I thought it did. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d688e0a..ce4b358b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -208,7 +208,7 @@ RUN set -x && \ # Append software versions RUN set -x && \ /usr/local/bin/ffmpeg -version && \ - FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | awk '1 == NR && "ffmpeg" == $1 { print $3; exit 0; } END { exit 1; }') && \ + FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | awk -v 'ev=31' '1 == NR && "ffmpeg" == $1 { print $3; ev=0; } END { exit ev; }') && \ test -n "${FFMPEG_VERSION}" && \ printf -- "ffmpeg_version = '%s'\n" "${FFMPEG_VERSION}" >> /app/common/third_party_versions.py