refactor: use hermetic tar

oci/private/tarball.bzl

@@ -76,12 +76,14 @@ def _tarball_impl(ctx):
 
  image = ctx.file.image
  tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name))
+ bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"]
  executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name))
  repo_tags = ctx.file.repo_tags
 
  substitutions = {
  "{{format}}": ctx.attr.format,
  "{{jq_path}}": jq.bin.path,
+ "{{tar}}": bsdtar.tarinfo.binary.path,
  "{{image_dir}}": image.path,
  "{{tarball_path}}": tarball.path,
  }
@@ -96,9 +98,15 @@ def _tarball_impl(ctx):
  substitutions = substitutions,
  )
 
+ # TODO(2.0): this oci_tarball rule should just produce an mtree manifest instead,
+ # and then the tar rule can be composed in the oci_tarball macro in defs.bzl.
+ # To make it a non-breaking change, call the tar program from within this action instead.
  ctx.actions.run(
  executable = util.maybe_wrap_launcher_for_windows(ctx, executable),
- inputs = [image, repo_tags, executable],
+ inputs = depset(
+ direct = [image, repo_tags, executable],
+ transitive = [bsdtar.default.files],
+ ),
  outputs = [tarball],
  tools = [jq.bin],
  mnemonic = "OCITarball",
@@ -131,6 +139,7 @@ oci_tarball = rule(
  toolchains = [
  "@bazel_tools//tools/sh:toolchain_type",
  "@aspect_bazel_lib//lib:jq_toolchain_type",
+ "@aspect_bazel_lib//lib:tar_toolchain_type",
  ],
  executable = True,
 )

oci/private/tarball.sh.tpl

@@ -2,19 +2,21 @@
 set -o pipefail -o errexit -o nounset
 
 readonly FORMAT="{{format}}"
-readonly STAGING_DIR=$(mktemp -d)
 readonly JQ="{{jq_path}}"
+readonly TAR="{{tar}}"
 readonly IMAGE_DIR="{{image_dir}}"
-readonly BLOBS_DIR="${STAGING_DIR}/blobs"
 readonly TARBALL_PATH="{{tarball_path}}"
 readonly REPOTAGS=($(cat "{{tags}}"))
 readonly INDEX_FILE="${IMAGE_DIR}/index.json"
 
-cp_f_with_mkdir() {
- SRC="$1"
- DST="$2"
- mkdir -p "$(dirname "${DST}")"
- cp -f "${SRC}" "${DST}"
+# Write tar manifest in mtree format
+# https://man.freebsd.org/cgi/man.cgi?mtree(8)
+# so that tar produces a deterministic output.
+mtree=$(mktemp)
+function add_to_tar() {
+ content=$1
+ tar_path=$2
+ echo >>"${mtree}" "${tar_path} uid=0 gid=0 mode=0755 time=1672560000 type=file content=${content}"
 }
 
 MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"')
@@ -46,36 +48,41 @@ if [[ "${FORMAT}" == "oci" ]]; then
  # Handle multi-architecture image indexes.
  # Ideally the toolchains we rely on would output these for us, but they don't seem to.
 
- echo -n '{"imageLayoutVersion": "1.0.0"}' > "${STAGING_DIR}/oci-layout"
+ layout_file=$(mktemp)
+ echo -n '{"imageLayoutVersion": "1.0.0"}' > "$layout_file"
+ add_to_tar "$layout_file" oci-layout
 
  INDEX_FILE_MANIFEST_DIGEST=$("${JQ}" -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"')
  INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}"
 
- cp_f_with_mkdir "${INDEX_FILE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${INDEX_FILE_MANIFEST_DIGEST}"
+ add_to_tar "${INDEX_FILE_MANIFEST_BLOB_PATH}" "blobs/${INDEX_FILE_MANIFEST_DIGEST}"
 
  IMAGE_MANIFESTS_DIGESTS=($("${JQ}" -r '.manifests[] | .digest | sub(":"; "/")' "${INDEX_FILE_MANIFEST_BLOB_PATH}"))
 
  for IMAGE_MANIFEST_DIGEST in "${IMAGE_MANIFESTS_DIGESTS[@]}"; do
  IMAGE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${IMAGE_MANIFEST_DIGEST}"
- cp_f_with_mkdir "${IMAGE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${IMAGE_MANIFEST_DIGEST}"
+ add_to_tar "${IMAGE_MANIFEST_BLOB_PATH}" "blobs/${IMAGE_MANIFEST_DIGEST}"
 
  CONFIG_DIGEST=$("${JQ}" -r '.config.digest | sub(":"; "/")' ${IMAGE_MANIFEST_BLOB_PATH})
  CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}"
- cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}"
+ add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}"
 
  LAYER_DIGESTS=$("${JQ}" -r '.layers | map(.digest | sub(":"; "/"))' "${IMAGE_MANIFEST_BLOB_PATH}")
  for LAYER_DIGEST in $("${JQ}" -r ".[]" <<< $LAYER_DIGESTS); do
- cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" ${BLOBS_DIR}/${LAYER_DIGEST}
+ add_to_tar "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" blobs/${LAYER_DIGEST}
  done
  done
 
 
  # Repeat the first manifest entry once per repo tag.
  repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}"
- "${JQ}" -r --arg repo_tags "$repotags" \
- '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}" > "${STAGING_DIR}/index.json"
+ index_json=$(mktemp)
+ "${JQ}" >"$index_json" \
+ -r --arg repo_tags "$repotags" \
+ '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}"
+ add_to_tar "$index_json" index.json
 
- tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout
+ ${TAR} --create --no-xattr --no-mac-metadata --file "${TARBALL_PATH}" "@${mtree}"
  exit 0
 fi
 
@@ -84,21 +91,25 @@ MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}"
 
 CONFIG_DIGEST=$(${JQ} -r '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH})
 CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}"
+add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}"
 
 LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH})
 
-cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}"
+add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}"
 
 for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do 
- cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz"
+ add_to_tar "${IMAGE_DIR}/blobs/${LAYER}" "blobs/${LAYER}.tar.gz"
 done
 
-
+manifest_json=$(mktemp)
 repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}"
-"${JQ}" -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \
- --arg repo_tags "$repotags" \
- --arg config "blobs/${CONFIG_DIGEST}" \
- --argjson layers "${LAYERS}" > "${STAGING_DIR}/manifest.json"
+"${JQ}" > "${manifest_json}" \
+ -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \
+ --arg repo_tags "$repotags" \
+ --arg config "blobs/${CONFIG_DIGEST}" \
+ --argjson layers "${LAYERS}"
+
+add_to_tar "${manifest_json}" "manifest.json"
 
-# TODO: https://github.com/bazel-contrib/rules_oci/issues/217
-tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" manifest.json blobs
+# We've created the manifest, now hand it off to tar to create our final output
+"${TAR}" --create --file "${TARBALL_PATH}" "@${mtree}"

oci/repositories.bzl

@@ -1,6 +1,6 @@
 """Repository rules for fetching external tools"""
 
-load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains")
+load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains")
 load("//oci/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo")
 load("//oci/private:versions.bzl", "CRANE_VERSIONS", "ZOT_VERSIONS")
 
@@ -113,6 +113,7 @@ def oci_register_toolchains(name, crane_version, zot_version = None, register =
  Should be True for WORKSPACE users, but false when used under bzlmod extension
  """
  register_jq_toolchains(register = register)
+ register_tar_toolchains(register = register)
  register_coreutils_toolchains(register = register)
  register_copy_to_directory_toolchains(register = register)