Merge pull request #300 from cevich/multiarch_mulligan

Improve & rename main build-push script

.cirrus.yml

@@ -97,9 +97,9 @@ container_images_task: &container_images
         #     TARGET_NAME: 'debian'
         #     DEST_FQIN: *fqin
     env: &image_env
-        # For quay.io/libpod namespace
-        REG_USERNAME: ENCRYPTED[de755aef351c501ee480231c24eae25b15e2b2a2b7c629f477c1d427fc5269e360bb358a53bd8914605bae588e99b52a]
-        REG_PASSWORD: ENCRYPTED[52268944bb0d6642c33efb1c5d7fb82d0c40f9e6988448de35827f9be2cc547c1383db13e8b21516dbd7a0a69a7ae536]
+        # For quay.io/libpod namespace, select FQINs only.
+        REG_USERNAME: ENCRYPTED[df4efe530b9a6a731cfea19233e395a5206d24dfac25e84329de035393d191e94ead8c39b373a0391fa025cab15470f8]
+        REG_PASSWORD: ENCRYPTED[255ec05057707c20237a6c7d15b213422779c534f74fe019b8ca565f635dba0e11035a034e533a6f39e146e7435d87b5]
     script: ci/make_container_images.sh;
     package_cache: &package_cache
         folder: "/tmp/automation_images_tmp/.cache/**"
@@ -477,7 +477,9 @@ test_build-push_task:
         disk: 200
         # More muscle to emulate multi-arch
         type: "n2-standard-4"
-    script: bash ./build-push/test.sh
+    script: |
+      bash ./build-push/.install.sh
+      bash ./build-push/test.sh
 
 
 # N/B: "latest" image produced after PR-merge (branch-push)

IMG_SFX

@@ -1 +1 @@
-20230822t185743z-f38f37d13
+20230918t183521z-f38f37d13

build-push/.install.sh

@@ -1,26 +1,25 @@
 #!/bin/bash
 
-# This script is intended to be used from two places only:
-# 1) When building the build-push VM image, to install the scripts as-is
-#    in a PR in order for CI testing to operate on them.
-# 2) From the autoupdate.sh script, when $BUILDPUSHAUTOUPDATED is unset
-#    or '0'.  This clones the latest repository to install (possibly)
-#    updated scripts.
+# This script is intended to be run from a task using a pre-existing
+# build-push VM image (having an image-suffix from the IMG_SFX file).
+# It's purpose is to install the latest version of the scripts in the
+# `bin` directory onto the system.
 #
 # WARNING: Use under any other circumstances will probably screw things up.
 
-if [[ -z "$BUILDPUSHAUTOUPDATED" ]];
-then
-    echo "This script must only be run under Packer or autoupdate.sh"
+# Common automation library pre-installed into the build-push VM
+if [[ -r /etc/automation_environment ]]; then
+    # Defines AUTOMATION_LIB_PATH and updates PATH
+    source /etc/automation_environment
+    source "$AUTOMATION_LIB_PATH/common_lib.sh"
+else
+    echo "ERROR: The common automation library has not been installed." > /dev/stderr
     exit 1
 fi
 
-source /etc/automation_environment
-source "$AUTOMATION_LIB_PATH/common_lib.sh"
+# Defined by common automation library
+# shellcheck disable=SC2154
+cd $(dirname "${BASH_SOURCE[0]}") || exit 1
 
-#shellcheck disable=SC2154
-cd $(dirname "$SCRIPT_FILEPATH") || exit 1
-# Must be installed into $AUTOMATION_LIB_PATH/../bin which is on $PATH
-cp ./bin/* $AUTOMATION_LIB_PATH/../bin/
-cp ./lib/* $AUTOMATION_LIB_PATH/
-chmod +x $AUTOMATION_LIB_PATH/../bin/*
+# Must be installed into $AUTOMATION_LIB_PATH/../bin which is also now on $PATH
+install -g root -o root -m 550 ./bin/* $AUTOMATION_LIB_PATH/../bin/

build-push/bin/main.sh → build-push/bin/containers_build_push.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 
-# This script is not intended for humans.  It should be run by automation
-# at the branch-level in automation for the skopeo, buildah, and podman
-# repositories.  It's purpose is to produce a multi-arch container image
-# based on the contents of context subdirectory.  At runtime, $PWD is assumed
-# to be the root of the cloned git repository.
+# This script is not intended for humans.  It should be run by secure
+# (maintainer-only) cron-like automation to service the skopeo, buildah,
+# and podman repositories.  It's purpose is to produce a multi-arch container
+# image based on the contents of a repository context subdirectory from their
+# respective 'main' branches.
 #
-# The first argument to the script, should be the URL of the git repository
-# in question.  Though at this time, this is only used for labeling the
-# resulting image.
+# The first argument to the script, should be the (clone) URL of the git repository
+# in question.  This is used to both retrieve the build context, as well as label
+# the produced images.
 #
 # The second argument to this script is the relative path to the build context
-# subdirectory.  The basename of this subdirectory may indicates the
-# image flavor (i.e. `upstream`, `testing`, or `stable`). Depending
+# subdirectory.  The basename of this subdirectory may (see next paragraph)
+# indicate the image flavor (i.e. `upstream`, `testing`, or `stable`). Depending
 # on this value, the image may be pushed to multiple container registries
 # under slightly different rules (see the next option).
 #
@@ -27,27 +27,27 @@ if [[ -r "/etc/automation_environment" ]]; then
     source /etc/automation_environment  # defines AUTOMATION_LIB_PATH
     #shellcheck disable=SC1090,SC2154
     source "$AUTOMATION_LIB_PATH/common_lib.sh"
-    #shellcheck source=../lib/autoupdate.sh
-    source "$AUTOMATION_LIB_PATH/autoupdate.sh"
 else
     echo "Expecting to find automation common library installed."
     exit 1
 fi
 
-# Careful: Changing the error message below could break auto-update test.
-if [[ "$#" -lt 2 ]]; then
-    #shellcheck disable=SC2145
-    die "Must be called with at least two arguments, got '$@'"
-fi
-
 if [[ -z $(type -P build-push.sh) ]]; then
     die "It does not appear that build-push.sh is installed properly"
 fi
 
-if ! [[ -d "$PWD/.git" ]]; then
-    die "The current directory ($PWD) does not appear to be the root of a git repo."
+if [[ -z "$1" ]]; then
+    die "Expecting a git repository URI as the first argument."
+fi
+
+# Careful: Changing the error message below could break auto-update test.
+if [[ "$#" -lt 2 ]]; then
+    #shellcheck disable=SC2145
+    die "Must be called with at least two arguments, got '$*'"
 fi
 
+req_env_vars CI
+
 # Assume transitive debugging state for build-push.sh if set
 if [[ "$(automation_version | cut -d '.' -f 1)" -ge 4 ]]; then
     # Valid for version 4.0.0 and above only
@@ -101,27 +101,67 @@ if ((DRYRUN)); then
     warn "Operating in dry-run mode with $_DRNOPUSH"
 fi
 
+# SCRIPT_PATH defined by automation library
+# shellcheck disable=SC2154
+CLONE_TMP=$(mktemp -p "" -d "tmp_${SCRIPT_FILENAME}_XXXX")
+trap "rm -rf '$CLONE_TMP'" EXIT
+
 ### MAIN
 
 declare -a build_args
 if [[ -n "$FLAVOR_NAME" ]]; then
-    build_args=(--build-arg "FLAVOR=$FLAVOR_NAME")
+    build_args=("--build-arg=FLAVOR=$FLAVOR_NAME")
 fi
 
-# Labels to add to all images
-# N/B: These won't show up in the manifest-list itself, only it's constituents.
-lblargs="\
-    --label=org.opencontainers.image.source=$REPO_URL \
-    --label=org.opencontainers.image.created=$(date -u --iso-8601=seconds)"
-dbg "lblargs=$lblargs"
+dbg "Cloning '$REPO_URL' into $CLONE_TMP"
+git clone --depth 1 "$REPO_URL" "$CLONE_TMP"
+cd "$CLONE_TMP"
+head_sha=$(git rev-parse HEAD)
+dbg "HEAD is $head_sha"
+
+req_env_vars CIRRUS_TASK_ID CIRRUS_CHANGE_IN_REPO CIRRUS_REPO_NAME
+
+# Labels to add to all images as per
+# https://specs.opencontainers.org/image-spec/annotations/?v=v1.0.1
+declare -a label_args
+
+# Use both labels and annotations since some older tools only support labels
+# CIRRUS_TASK_ID provided by CI and verified non-empty
+# shellcheck disable=SC2154
+for arg in "--label" "--annotation"; do
+  label_args+=(\
+    "$arg=org.opencontainers.image.source=$REPO_URL"
+    "$arg=org.opencontainers.image.revision=$head_sha"
+    "$arg=org.opencontainers.image.created=$(date -u --iso-8601=seconds)"
+    "$arg=org.opencontainers.image.documentation=${REPO_URL%.git}/tree/$CTX_SUB/README.md"
+    "$arg=org.opencontainers.image.authors=podman@lists.podman.io"
+  )
+
+  # Perhaps slightly outside the intended purpose, but it kind of fits, and may help
+  # somebody ascertain provenance a little better.  Note: Even if the console logs
+  # are blank, the Cirrus-CI GraphQL API keeps build and task metadata for years.
+  label_args+=(\
+    "$arg=org.opencontainers.image.url=https://cirrus-ci.com/task/$CIRRUS_TASK_ID"
+  )
+
+  # Definitely not any official spec., but offers a quick reference to exactly what produced
+  # the images and it's current signature.
+  label_args+=(\
+    "$arg=built.by.repo=${CIRRUS_REPO_NAME}"
+    "$arg=built.by.commit=${CIRRUS_CHANGE_IN_REPO}"
+    "$arg=built.by.exec=$(basename ${BASH_SOURCE[0]})"
+    "$arg=built.by.digest=sha256:$(sha256sum<${BASH_SOURCE[0]} | awk '{print $1}')"
+  )
+done
 
 modcmdarg="tag_version.sh $FLAVOR_NAME"
 
-# For stable images, the version number of the command is needed for tagging.
+# For stable images, the version number of the command is needed for tagging and labeling.
 if [[ "$FLAVOR_NAME" == "stable" ]]; then
     # only native arch is needed to extract the version
-    dbg "Building local-arch image to extract stable version number"
-    podman build -t $REPO_FQIN "${build_args[@]}" ./$CTX_SUB
+    dbg "Building temporary local-arch image to extract stable version number"
+    FQIN_TMP="$REPO_NAME:temp"
+    showrun podman build -t $FQIN_TMP "${build_args[@]}" ./$CTX_SUB
 
     case "$REPO_NAME" in
         skopeo) version_cmd="--version" ;;
@@ -131,42 +171,42 @@ if [[ "$FLAVOR_NAME" == "stable" ]]; then
         *) die "Unknown/unsupported repo '$REPO_NAME'" ;;
     esac
 
-    pvcmd="podman run -i --rm $REPO_FQIN $version_cmd"
+    pvcmd="podman run -i --rm $FQIN_TMP $version_cmd"
     dbg "Extracting version with command: $pvcmd"
     version_output=$($pvcmd)
-    dbg "version output:
-    $version_output
-    "
+    dbg "version output: '$version_output'"
     img_cmd_version=$(awk -r -e '/^.+ version /{print $3}' <<<"$version_output")
     dbg "parsed version: $img_cmd_version"
     test -n "$img_cmd_version"
-    lblargs="$lblargs --label=org.opencontainers.image.version=$img_cmd_version"
-    # Prevent temporary build colliding with multi-arch manifest list (built next)
-    # but preserve image (by ID) for use as cache.
-    dbg "Un-tagging $REPO_FQIN"
-    podman untag $REPO_FQIN
+
+    label_args+=("--label=org.opencontainers.image.version=$img_cmd_version"
+                 "--annotation=org.opencontainers.image.version=$img_cmd_version")
 
     # tag-version.sh expects this arg. when FLAVOR_NAME=stable
     modcmdarg+=" $img_cmd_version"
 
+    dbg "Building stable-flavor manifest-list '$_REG/containers/$REPO_NAME'"
+
     # Stable images get pushed to 'containers' namespace as latest & version-tagged
-    build-push.sh \
+    showrun build-push.sh \
         $_DRNOPUSH \
-        --arches=$ARCHES \
+        --arches="$ARCHES" \
         --modcmd="$modcmdarg" \
-        $_REG/containers/$REPO_NAME \
-        ./$CTX_SUB \
-        $lblargs \
-        "${build_args[@]}"
+        "$_REG/containers/$REPO_NAME" \
+        "./$CTX_SUB" \
+        "${build_args[@]}" \
+        "${label_args[@]}"
 fi
 
+dbg "Building manifest-list '$REPO_FQIN'"
+
 # All images are pushed to quay.io/<reponame>, both
 # latest and version-tagged (if available).
-build-push.sh \
+showrun build-push.sh \
     $_DRNOPUSH \
-    --arches=$ARCHES \
+    --arches="$ARCHES" \
     --modcmd="$modcmdarg" \
-    $REPO_FQIN \
-    ./$CTX_SUB \
-    $lblargs \
-    "${build_args[@]}"
+    "$REPO_FQIN" \
+    "./$CTX_SUB" \
+    "${build_args[@]}" \
+    "${label_args[@]}"

build-push/bin/tag_version.sh

@@ -3,7 +3,11 @@
 # This script is not intended for humans.  It should only be referenced
 # as an argument to the build-push.sh `--modcmd` option.  It's purpose
 # is to ensure stable images are re-tagged with a verison-number
-# cooresponding to the included tool's version.
+# using a specific scheme, cooresponding to the included tool's version.
+# It expects two arguments, the first is the "flavor" of the image.
+# Typically 'upstream', 'testing', or 'stable'.  The second argument
+# is optional, when provided it should be the container-image's tool
+# version number (e.g. podman version).
 
 set -eo pipefail
 
@@ -20,33 +24,36 @@ fi
 req_env_vars SCRIPT_FILENAME SCRIPT_FILEPATH RUNTIME PLATFORMOS FQIN CONTEXT \
              PUSH ARCHES REGSERVER NAMESPACE IMGNAME MODCMD
 
-if [[ "$#" -ge 1 ]]; then
-    FLAVOR_NAME="$1"  # upstream, testing, or stable
+if [[ "$#" -lt 1 ]]; then
+    # Defined by common automation library
+    # shellcheck disable=SC2154
+    die "$SCRIPT_FILENAME expects at least one argument"
 fi
 
 if [[ "$#" -ge 2 ]]; then
-    # Enforce all version-tags start with a 'v'
-    VERSION="v${2#v}"  # output of $version_cmd
+    FLAVOR_NAME="$1"
+    # Version is optional
+    unset VERSION
+    [[ -z "$2" ]] || \
+        VERSION="v${2#v}"
 fi
 
 if [[ -z "$FLAVOR_NAME" ]]; then
     # Defined by common_lib.sh
     # shellcheck disable=SC2154
-    warn "$SCRIPT_FILENAME passed empty flavor-name argument (optional)."
+    warn "$SCRIPT_FILENAME passed empty flavor-name argument."
 elif [[ -z "$VERSION" ]]; then
-    warn "$SCRIPT_FILENAME received empty version argument (req. for FLAVOR_NAME=stable)."
+    warn "$SCRIPT_FILENAME received empty version argument."
 fi
 
 # shellcheck disable=SC2154
-dbg "Mod-command operating on $FQIN in '$FLAVOR_NAME' flavor"
+dbg "$SCRIPT_FILENAME operating on '$FLAVOR_NAME' flavor of '$FQIN' with tool version '$VERSION' (optional)"
 
 if [[ "$FLAVOR_NAME" == "stable" ]]; then
     # Stable images must all be tagged with a version number.
     # Confirm this value is passed in by caller.
-    req_env_vars VERSION
-    VERSION=v${VERSION#v}
     if grep -E -q '^v[0-9]+\.[0-9]+\.[0-9]+'<<<"$VERSION"; then
-        msg "Found image command version '$VERSION'"
+        msg "Using provided image command version '$VERSION'"
     else
         die "Encountered unexpected/non-conforming version '$VERSION'"
     fi
@@ -65,5 +72,5 @@ if [[ "$FLAVOR_NAME" == "stable" ]]; then
     $RUNTIME tag $FQIN:latest $FQIN:$x_ver
     msg "Successfully tagged $FQIN:$x_ver"
 else
-    warn "$SCRIPT_FILENAME not version-tagging for '$FLAVOR_NAME' stage of '$FQIN'"
+    warn "$SCRIPT_FILENAME not version-tagging for '$FLAVOR_NAME' flavor'$FQIN'"
 fi