Add infrastructure for caching downloads in separate image

ShareX · Jun 18, 2023 · c3fec1b · c3fec1b
1 parent 78191a7
commit c3fec1b
Show file tree

Hide file tree

Showing 109 changed files with 631 additions and 204 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,4 @@
-/Dockerfile
+/Dockerfile*
 /ffbuild/
 /artifacts/
 /.cache/
diff --git a/generate.sh b/generate.sh
@@ -3,13 +3,71 @@ set -e
 cd "$(dirname "$0")"
 source util/vars.sh
 
-rm -f Dockerfile
+rm -f Dockerfile Dockerfile.{dl,final,dl.final}
 
 layername() {
     printf "layer-"
     basename "$1" | sed 's/.sh$//'
 }
 
+to_df() {
+    _of="${TODF:-Dockerfile}"
+    printf "$@" >> "$_of"
+    echo >> "$_of"
+}
+
+###
+### Generate download Dockerfile
+###
+
+exec_dockerstage_dl() {
+    SCRIPT="$1"
+    (
+        SELF="$SCRIPT"
+        SELFLAYER="$(layername "$STAGE")"
+        source "$SCRIPT"
+        ffbuild_dockerstage_dl || exit $?
+        TODF="Dockerfile.dl.final" ffbuild_dockerlayer_dl || exit $?
+    )
+}
+
+export TODF="Dockerfile.dl"
+
+to_df "FROM ${REGISTRY}/${REPO}/base:latest AS base"
+to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
+
+PREVLAYER="base"
+for ID in $(ls -1d scripts.d/??-* | sed -s 's|^.*/\(..\).*|\1|' | sort -u); do
+    LAYER="layer-$ID"
+
+    for STAGE in scripts.d/$ID-*; do
+        if [[ -f "$STAGE" ]]; then
+            to_df "FROM $PREVLAYER AS $(layername "$STAGE")"
+            exec_dockerstage_dl "$STAGE"
+        else
+            for STAGE in "${STAGE}"/??-*; do
+                to_df "FROM $PREVLAYER AS $(layername "$STAGE")"
+                exec_dockerstage_dl "$STAGE"
+            done
+        fi
+    done
+done
+
+to_df "FROM base AS intermediate"
+cat Dockerfile.dl.final >> "$TODF"
+rm Dockerfile.dl.final
+
+to_df "FROM base"
+to_df "COPY --from=intermediate \$FFBUILD_DLDIR/. \$FFBUILD_DLDIR"
+
+if [[ "$TARGET" == "dl" && "$VARIANT" == "only" ]]; then
+    exit 0
+fi
+
+###
+### Generate main Dockerfile
+###
+
 exec_dockerstage() {
     SCRIPT="$1"
     (
@@ -20,11 +78,7 @@ exec_dockerstage() {
     )
 }
 
-to_df() {
-    _of="${TODF:-Dockerfile}"
-    printf "$@" >> "$_of"
-    echo >> "$_of"
-}
+export TODF="Dockerfile"
 
 to_df "FROM ${REGISTRY}/${REPO}/base-${TARGET}:latest AS base"
 to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"

diff --git a/images/base/Dockerfile b/images/base/Dockerfile
@@ -1,7 +1,10 @@
 FROM ubuntu:23.04
 
+ENV FFBUILD_DLDIR="/opt/ffdl"
+
 ENV DEBIAN_FRONTEND noninteractive
 RUN \
+    mkdir -p "$FFBUILD_DLDIR" && \
     apt-get -y update && \
     apt-get -y dist-upgrade && \
     apt-get -y install build-essential yasm nasm \

diff --git a/images/base/run_stage.sh b/images/base/run_stage.sh
@@ -11,5 +11,12 @@ export RAW_LDFLAGS="$LDFLAGS"
 mkdir -p /stage
 source "$1"
 cd /stage
-ffbuild_dockerbuild
+if [[ -n "$3" ]]; then
+    cd "$3"
+fi
+if [[ -z "$2" ]]; then
+    ffbuild_dockerbuild
+else
+    "$2"
+fi
 rm -rf /stage "$FFBUILD_PREFIX"/bin
diff --git a/makeimage.sh b/makeimage.sh
@@ -18,13 +18,14 @@ docker buildx inspect ffbuilder &>/dev/null || docker buildx create \
     --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=-1 \
     --driver-opt env.BUILDKIT_STEP_LOG_MAX_SPEED=-1
 
-if [[ -z "$QUICKBUILD" ]]; then
-    docker container inspect ffbuildreg &>/dev/null || \
-        docker run --rm -d -p 127.0.0.1:0:5000 --name ffbuildreg registry:2
-    LOCAL_REG_PORT="$(docker container inspect --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}}{{end}}' ffbuildreg)"
-    LOCAL_ROOT="127.0.0.1:${LOCAL_REG_PORT}/local"
-    trap "rm -f '$TMPCFG'; docker container stop ffbuildreg" EXIT
+docker container inspect ffbuildreg &>/dev/null || \
+    docker run --rm -d -p 127.0.0.1:64647:5000 --name ffbuildreg registry:2
+LOCAL_REG_PORT="$(docker container inspect --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}}{{end}}' ffbuildreg)"
+LOCAL_ROOT="127.0.0.1:${LOCAL_REG_PORT}/local"
+
+export REGISTRY_OVERRIDE_DL="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY_DL="local"
 
+if [[ -z "$QUICKBUILD" ]]; then
     if grep "FROM.*base.*" "images/base-${TARGET}/Dockerfile" >/dev/null 2>&1; then
         docker buildx --builder ffbuilder build \
             --cache-from=type=local,src=.cache/"${BASE_IMAGE/:/_}" \
@@ -38,14 +39,20 @@ if [[ -z "$QUICKBUILD" ]]; then
         --push --tag "${LOCAL_ROOT}/base-${TARGET}:latest" \
         --build-arg GH_REPO="$LOCAL_ROOT" "images/base-${TARGET}"
 
-    export REGISTRY_OVERRIDE="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY="local"
+    export REGISTRY_OVERRIDE="$REGISTRY_OVERRIDE_DL" GITHUB_REPOSITORY="$GITHUB_REPOSITORY_DL"
 fi
 
 ./generate.sh "$TARGET" "$VARIANT" "${ADDINS[@]}"
 
+docker buildx --builder ffbuilder build -f Dockerfile.dl \
+    --cache-from=type=local,src=.cache/"${DL_IMAGE/:/_}" \
+    --cache-to=type=local,mode=max,dest=.cache/"${DL_IMAGE/:/_}" \
+    --push --tag "${LOCAL_ROOT}/dl_cache:latest" .
+
 docker buildx --builder ffbuilder build \
     --cache-from=type=local,src=.cache/"${IMAGE/:/_}" \
     --cache-to=type=local,mode=max,dest=.cache/"${IMAGE/:/_}" \
     --load --tag "$IMAGE" .
 
+docker container stop ffbuildreg
 docker buildx rm -f ffbuilder
diff --git a/scripts.d/10-mingw-std-threads.sh b/scripts.d/10-mingw-std-threads.sh
@@ -8,9 +8,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" mingw-std-threads
-    cd mingw-std-threads
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/mingw-std-threads
 
     mkdir -p "$FFBUILD_PREFIX"/include
     cp *.h "$FFBUILD_PREFIX"/include

diff --git a/scripts.d/10-mingw.sh b/scripts.d/10-mingw.sh
@@ -17,10 +17,14 @@ ffbuild_dockerfinal() {
     to_df "COPY --from=${PREVLAYER} /opt/mingw/. /"
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     retry-tool sh -c "rm -rf mingw && git clone '$SCRIPT_REPO' mingw"
     cd mingw
     git checkout "$SCRIPT_COMMIT"
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR/mingw"
 
     cd mingw-w64-headers
 

diff --git a/scripts.d/10-xorg-macros.sh b/scripts.d/10-xorg-macros.sh
@@ -13,9 +13,12 @@ ffbuild_dockerlayer() {
     to_df "COPY --from=${SELFLAYER} \$FFBUILD_PREFIX/share/aclocal/. /usr/share/aclocal"
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" xorg-macros
-    cd xorg-macros
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/xorg-macros
 
     autoreconf -i
     ./configure --prefix="$FFBUILD_PREFIX"

diff --git a/scripts.d/20-libiconv.sh b/scripts.d/20-libiconv.sh
@@ -7,10 +7,15 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     retry-tool sh -c "rm -rf iconv && git clone '$SCRIPT_REPO' iconv"
     cd iconv
     git checkout "$SCRIPT_COMMIT"
+    retry-tool ./autopull.sh --one-time
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/iconv
 
     retry-tool ./autopull.sh --one-time
     (unset CC CFLAGS GMAKE && ./autogen.sh)

diff --git a/scripts.d/20-libxml2.sh b/scripts.d/20-libxml2.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" libxml2
-    cd libxml2
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/libxml2
 
     local myconf=(
         --prefix="$FFBUILD_PREFIX"

diff --git a/scripts.d/20-zlib.sh b/scripts.d/20-zlib.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" zlib
-    cd zlib
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/zlib
 
     local myconf=(
         --prefix="$FFBUILD_PREFIX"

diff --git a/scripts.d/25-fftw3.sh b/scripts.d/25-fftw3.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" fftw3
-    cd fftw3
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/fftw3
 
     local myconf=(
         --prefix="$FFBUILD_PREFIX"

diff --git a/scripts.d/25-freetype.sh b/scripts.d/25-freetype.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" freetype
-    cd freetype
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/freetype
 
     ./autogen.sh
 

diff --git a/scripts.d/25-fribidi.sh b/scripts.d/25-fribidi.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" fribidi
-    cd fribidi
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/fribidi
 
     mkdir build && cd build
 

diff --git a/scripts.d/25-gmp.sh b/scripts.d/25-gmp.sh
@@ -8,10 +8,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     retry-tool check-wget "gmp.tar.xz" "$SCRIPT_URL" "$SCRIPT_SHA512"
+}
 
-    tar xaf "gmp.tar.xz"
+ffbuild_dockerbuild() {
+    tar xaf "$FFBUILD_DLDIR"/gmp.tar.xz
     cd "gmp-$SCRIPT_VERSION"
 
     local myconf=(

diff --git a/scripts.d/25-libogg.sh b/scripts.d/25-libogg.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" ogg
-    cd ogg
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/ogg
 
     ./autogen.sh
 

diff --git a/scripts.d/25-openssl.sh b/scripts.d/25-openssl.sh
@@ -8,10 +8,14 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" openssl
     cd openssl
     git submodule update --init --recursive --depth=1
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/openssl
 
     local myconf=(
         threads

diff --git a/scripts.d/25-xz.sh b/scripts.d/25-xz.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" xz
-    cd xz
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/xz
 
     ./autogen.sh --no-po4a
 

diff --git a/scripts.d/35-fontconfig.sh b/scripts.d/35-fontconfig.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" fc
-    cd fc
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/fc
 
     ./autogen.sh --noconf
 

diff --git a/scripts.d/45-harfbuzz.sh b/scripts.d/45-harfbuzz.sh
@@ -7,9 +7,12 @@ ffbuild_enabled() {
     return 0
 }
 
-ffbuild_dockerbuild() {
+ffbuild_dockerdl() {
     git-mini-clone "$SCRIPT_REPO" "$SCRIPT_COMMIT" harfbuzz
-    cd harfbuzz
+}
+
+ffbuild_dockerbuild() {
+    cd "$FFBUILD_DLDIR"/harfbuzz
 
     local myconf=(
         --prefix="$FFBUILD_PREFIX"