diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index fa23ae3e..0b4bf63d 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -76,12 +76,14 @@ def _tarball_impl(ctx): image = ctx.file.image tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name)) + bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"] executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name)) repo_tags = ctx.file.repo_tags substitutions = { "{{format}}": ctx.attr.format, "{{jq_path}}": jq.bin.path, + "{{tar}}": bsdtar.tarinfo.binary.path, "{{image_dir}}": image.path, "{{tarball_path}}": tarball.path, } @@ -96,9 +98,15 @@ def _tarball_impl(ctx): substitutions = substitutions, ) + # TODO(2.0): this oci_tarball rule should just produce an mtree manifest instead, + # and then the tar rule can be composed in the oci_tarball macro in defs.bzl. + # To make it a non-breaking change, call the tar program from within this action instead. ctx.actions.run( executable = util.maybe_wrap_launcher_for_windows(ctx, executable), - inputs = [image, repo_tags, executable], + inputs = depset( + direct = [image, repo_tags, executable], + transitive = [bsdtar.default.files], + ), outputs = [tarball], tools = [jq.bin], mnemonic = "OCITarball", @@ -131,6 +139,7 @@ oci_tarball = rule( toolchains = [ "@bazel_tools//tools/sh:toolchain_type", "@aspect_bazel_lib//lib:jq_toolchain_type", + "@aspect_bazel_lib//lib:tar_toolchain_type", ], executable = True, ) diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index a25209df..16262129 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -2,19 +2,21 @@ set -o pipefail -o errexit -o nounset readonly FORMAT="{{format}}" -readonly STAGING_DIR=$(mktemp -d) readonly JQ="{{jq_path}}" +readonly TAR="{{tar}}" readonly IMAGE_DIR="{{image_dir}}" -readonly BLOBS_DIR="${STAGING_DIR}/blobs" readonly TARBALL_PATH="{{tarball_path}}" readonly REPOTAGS=($(cat "{{tags}}")) readonly INDEX_FILE="${IMAGE_DIR}/index.json" -cp_f_with_mkdir() { - SRC="$1" - DST="$2" - mkdir -p "$(dirname "${DST}")" - cp -f "${SRC}" "${DST}" +# Write tar manifest in mtree format +# https://man.freebsd.org/cgi/man.cgi?mtree(8) +# so that tar produces a deterministic output. +mtree=$(mktemp) +function add_to_tar() { + content=$1 + tar_path=$2 + echo >>"${mtree}" "${tar_path} uid=0 gid=0 mode=0755 time=1672560000 type=file content=${content}" } MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') @@ -46,36 +48,41 @@ if [[ "${FORMAT}" == "oci" ]]; then # Handle multi-architecture image indexes. # Ideally the toolchains we rely on would output these for us, but they don't seem to. - echo -n '{"imageLayoutVersion": "1.0.0"}' > "${STAGING_DIR}/oci-layout" + layout_file=$(mktemp) + echo -n '{"imageLayoutVersion": "1.0.0"}' > "$layout_file" + add_to_tar "$layout_file" oci-layout INDEX_FILE_MANIFEST_DIGEST=$("${JQ}" -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}" - cp_f_with_mkdir "${INDEX_FILE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${INDEX_FILE_MANIFEST_DIGEST}" + add_to_tar "${INDEX_FILE_MANIFEST_BLOB_PATH}" "blobs/${INDEX_FILE_MANIFEST_DIGEST}" IMAGE_MANIFESTS_DIGESTS=($("${JQ}" -r '.manifests[] | .digest | sub(":"; "/")' "${INDEX_FILE_MANIFEST_BLOB_PATH}")) for IMAGE_MANIFEST_DIGEST in "${IMAGE_MANIFESTS_DIGESTS[@]}"; do IMAGE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${IMAGE_MANIFEST_DIGEST}" - cp_f_with_mkdir "${IMAGE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${IMAGE_MANIFEST_DIGEST}" + add_to_tar "${IMAGE_MANIFEST_BLOB_PATH}" "blobs/${IMAGE_MANIFEST_DIGEST}" CONFIG_DIGEST=$("${JQ}" -r '.config.digest | sub(":"; "/")' ${IMAGE_MANIFEST_BLOB_PATH}) CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" - cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" + add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" LAYER_DIGESTS=$("${JQ}" -r '.layers | map(.digest | sub(":"; "/"))' "${IMAGE_MANIFEST_BLOB_PATH}") for LAYER_DIGEST in $("${JQ}" -r ".[]" <<< $LAYER_DIGESTS); do - cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" ${BLOBS_DIR}/${LAYER_DIGEST} + add_to_tar "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" blobs/${LAYER_DIGEST} done done # Repeat the first manifest entry once per repo tag. repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" - "${JQ}" -r --arg repo_tags "$repotags" \ - '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}" > "${STAGING_DIR}/index.json" + index_json=$(mktemp) + "${JQ}" >"$index_json" \ + -r --arg repo_tags "$repotags" \ + '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}" + add_to_tar "$index_json" index.json - tar --directory "${STAGING_DIR}" --create --no-xattr --file "${TARBALL_PATH}" index.json blobs oci-layout + ${TAR} --create --no-xattr --no-mac-metadata --file "${TARBALL_PATH}" "@${mtree}" exit 0 fi @@ -84,21 +91,25 @@ MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}" CONFIG_DIGEST=$(${JQ} -r '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" +add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) -cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" +add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do - cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz" + add_to_tar "${IMAGE_DIR}/blobs/${LAYER}" "blobs/${LAYER}.tar.gz" done - +manifest_json=$(mktemp) repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" -"${JQ}" -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \ - --arg repo_tags "$repotags" \ - --arg config "blobs/${CONFIG_DIGEST}" \ - --argjson layers "${LAYERS}" > "${STAGING_DIR}/manifest.json" +"${JQ}" > "${manifest_json}" \ + -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \ + --arg repo_tags "$repotags" \ + --arg config "blobs/${CONFIG_DIGEST}" \ + --argjson layers "${LAYERS}" + +add_to_tar "${manifest_json}" "manifest.json" -# TODO: https://github.com/bazel-contrib/rules_oci/issues/217 -tar --directory "${STAGING_DIR}" --create --no-xattr --file "${TARBALL_PATH}" manifest.json blobs +# We've created the manifest, now hand it off to tar to create our final output +"${TAR}" --create --no-xattr --no-mac-metadata --file "${TARBALL_PATH}" "@${mtree}" diff --git a/oci/repositories.bzl b/oci/repositories.bzl index a8ace689..83af5c14 100644 --- a/oci/repositories.bzl +++ b/oci/repositories.bzl @@ -1,6 +1,6 @@ """Repository rules for fetching external tools""" -load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains") +load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains") load("//oci/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo") load("//oci/private:versions.bzl", "CRANE_VERSIONS", "ZOT_VERSIONS") @@ -113,6 +113,7 @@ def oci_register_toolchains(name, crane_version, zot_version = None, register = Should be True for WORKSPACE users, but false when used under bzlmod extension """ register_jq_toolchains(register = register) + register_tar_toolchains(register = register) register_coreutils_toolchains(register = register) register_copy_to_directory_toolchains(register = register)