refactor: make pinning instructions a warning mesage

WORKSPACE

@@ -113,3 +113,12 @@ new_local_repository(
  build_file = "//examples/attest_external:BUILD.template",
  path = "examples/attest_external/workspace",
 )
+
+oci_pull(
+ name = "thesayyn_debian",
+ image = "index.docker.io/library/debian",
+ platforms = ["linux/arm64"],
+ # Don't make a distroless_python_unpinned repo and print a warning about the tag
+ reproducible = False,
+ tag = "latest",
+)

docs/pull.md

@@ -43,7 +43,8 @@ oci_image(
 ## oci_alias
 
 <pre>
-oci_alias(<a href="#oci_alias-name">name</a>, <a href="#oci_alias-platforms">platforms</a>, <a href="#oci_alias-repo_mapping">repo_mapping</a>)
+oci_alias(<a href="#oci_alias-name">name</a>, <a href="#oci_alias-identifier">identifier</a>, <a href="#oci_alias-image">image</a>, <a href="#oci_alias-platforms">platforms</a>, <a href="#oci_alias-repo_mapping">repo_mapping</a>, <a href="#oci_alias-reproducible">reproducible</a>, <a href="#oci_alias-single_platform">single_platform</a>,
+ <a href="#oci_alias-toolchain_name">toolchain_name</a>)
 </pre>
 
 
@@ -54,8 +55,13 @@ oci_alias(<a href="#oci_alias-name">name</a>, <a href="#oci_alias-platforms">pla
 | Name | Description | Type | Mandatory | Default |
 | :------------- | :------------- | :------------- | :------------- | :------------- |
 | <a id="oci_alias-name"></a>name | A unique name for this repository. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
+| <a id="oci_alias-identifier"></a>identifier | - | String | optional | "" |
+| <a id="oci_alias-image"></a>image | The name of the image we are fetching, e.g. gcr.io/distroless/static | String | required | |
 | <a id="oci_alias-platforms"></a>platforms | - | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: Label -> String</a> | optional | {} |
 | <a id="oci_alias-repo_mapping"></a>repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.&lt;p&gt;For example, an entry <code>"@foo": "@bar"</code> declares that, for any time this repository depends on <code>@foo</code> (such as a dependency on <code>@foo//some:target</code>, it should actually resolve that dependency within globally-declared <code>@bar</code> (<code>@bar//some:target</code>). | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | required | |
+| <a id="oci_alias-reproducible"></a>reproducible | Set to False to silence the warning about reproducibility when using <code>tag</code> | Boolean | optional | True |
+| <a id="oci_alias-single_platform"></a>single_platform | - | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="oci_alias-toolchain_name"></a>toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" |
 
 
 <a id="#oci_pull_rule"></a>
@@ -81,28 +87,6 @@ oci_pull_rule(<a href="#oci_pull_rule-name">name</a>, <a href="#oci_pull_rule-id
 | <a id="oci_pull_rule-toolchain_name"></a>toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" |
 
 
-<a id="#pin_tag"></a>
-
-## pin_tag
-
-<pre>
-pin_tag(<a href="#pin_tag-name">name</a>, <a href="#pin_tag-image">image</a>, <a href="#pin_tag-repo_mapping">repo_mapping</a>, <a href="#pin_tag-tag">tag</a>, <a href="#pin_tag-toolchain_name">toolchain_name</a>)
-</pre>
-
-
-
-**ATTRIBUTES**
-
-
-| Name | Description | Type | Mandatory | Default |
-| :------------- | :------------- | :------------- | :------------- | :------------- |
-| <a id="pin_tag-name"></a>name | A unique name for this repository. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
-| <a id="pin_tag-image"></a>image | The name of the image we are fetching, e.g. <code>gcr.io/distroless/static</code> | String | required | |
-| <a id="pin_tag-repo_mapping"></a>repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.&lt;p&gt;For example, an entry <code>"@foo": "@bar"</code> declares that, for any time this repository depends on <code>@foo</code> (such as a dependency on <code>@foo//some:target</code>, it should actually resolve that dependency within globally-declared <code>@bar</code> (<code>@bar//some:target</code>). | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | required | |
-| <a id="pin_tag-tag"></a>tag | The tag being used, e.g. <code>latest</code> | String | required | |
-| <a id="pin_tag-toolchain_name"></a>toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" |
-
-
 <a id="#oci_pull"></a>
 
 ## oci_pull

oci/BUILD.bazel

@@ -41,7 +41,6 @@ bzl_library(
  visibility = ["//visibility:public"],
  deps = [
  "@aspect_bazel_lib//lib:base64",
- "@aspect_bazel_lib//lib:paths",
  "@aspect_bazel_lib//lib:repo_utils",
  ],
 )

oci/pull.bzl

@@ -36,7 +36,6 @@ oci_image(
 ```
 """
 
-load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION")
 load("@aspect_bazel_lib//lib:base64.bzl", "base64")
 load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils")
 
@@ -168,8 +167,34 @@ Running one of `podman login`, `docker login`, `crane login` may help.
 
 # OCI Image Media Types
 # Spec: https://github.com/distribution/distribution/blob/main/docs/spec/manifest-v2-2.md#media-types
-_MANIFEST_TYPE = "application/vnd.docker.distribution.manifest.v2+json"
-_MANIFEST_LIST_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"
+OCI_IMAGE_INDEX = "application/vnd.oci.image.index.v1+json"
+DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
+OCI_MANIFEST_SCHEMA1 = "application/vnd.oci.image.manifest.v1+json"
+DOCKER_MANIFEST_SCHEMA2 = "application/vnd.docker.distribution.manifest.v2+json"
+
+def _is_image_index(descriptor):
+ media_type = descriptor["mediaType"]
+ return media_type == OCI_IMAGE_INDEX or media_type == DOCKER_MANIFEST_LIST
+
+def _is_image(descriptor):
+ media_type = descriptor["mediaType"]
+ return media_type == OCI_MANIFEST_SCHEMA1 or media_type == DOCKER_MANIFEST_SCHEMA2
+
+def _parse_reference(reference):
+ firstslash = reference.find("/")
+ registry = reference[:firstslash]
+ repository = reference[firstslash + 1:]
+ return registry, repository
+
+def _is_tag(str):
+ return str.find(":") == -1
+
+def _trim_hash_algorithm(identifier):
+ "Optionally remove the sha256: prefix from identifier, if present"
+ parts = identifier.split(":", 1)
+ if len(parts) != 2:
+ return identifier
+ return parts[1]
 
 def _parse_reference(reference):
  firstslash = reference.find("/")
@@ -314,18 +339,20 @@ def _oci_pull_impl(rctx):
  mf_file = _trim_hash_algorithm(rctx.attr.identifier)
  mf, mf_len = _download_manifest(rctx, rctx.attr.identifier, mf_file)
 
- if mf["mediaType"] == _MANIFEST_TYPE:
+ if _is_image(mf):
  if rctx.attr.platform:
  fail("{} is a single-architecture image, so attribute 'platform' should not be set.".format(rctx.attr.image))
  image_mf_file = mf_file
  image_mf = mf
  image_mf_len = mf_len
  image_digest = rctx.attr.identifier
- elif mf["mediaType"] == _MANIFEST_LIST_TYPE:
+ elif _is_image_index(mf):
  # extra download to get the manifest for the selected arch
  if not rctx.attr.platform:
  fail("{} is a multi-architecture image, so attribute 'platform' is required.".format(rctx.attr.image))
- matching_mf = _find_platform_manifest(rctx, mf)
+ matching_mf = _find_platform_manifest(mf, rctx.attr.platform)
+ if not matching_mf:
+ fail("No matching manifest found in image {} for platform {}".format(rctx.attr.image, rctx.attr.platform))
  image_digest = matching_mf["digest"]
  image_mf_file = _trim_hash_algorithm(image_digest)
  image_mf, image_mf_len = _download_manifest(rctx, image_digest, image_mf_file)
@@ -407,7 +434,7 @@ oci_pull_rule = repository_rule(
  ],
 )
 
-_alias_target = """\
+_MULTI_PLATFORM_IMAGE_ALIAS = """\
 alias(
  name = "{name}",
  actual = select(
@@ -417,96 +444,80 @@ alias(
 )
 """
 
-def _oci_alias_impl(rctx):
- rctx.file("BUILD.bazel", content = _alias_target.format(
- name = rctx.attr.name,
- platform_map = {str(k): v for k, v in rctx.attr.platforms.items()},
- ))
-
-oci_alias = repository_rule(
- implementation = _oci_alias_impl,
- attrs = {
- "platforms": attr.label_keyed_string_dict(),
- },
+_SINGLE_PLATFORM_IMAGE_ALIAS = """\
+alias(
+ name = "{name}",
+ actual = "@{original}//:{original}",
+ visibility = ["//visibility:public"],
 )
+"""
 
-_latest_build = """\
-load("@aspect_bazel_lib//lib:jq.bzl", "jq")
-load("@bazel_skylib//rules:write_file.bzl", "write_file")
+def _coreutils_label(rctx):
+ return Label("@coreutils_{}//:coreutils".format(repo_utils.platform(rctx)))
 
-jq(
- name = "platforms",
- srcs = ["manifest_list.json"],
- filter = "[(.manifests // [])[] | .platform | .os + \\"/\\" + .architecture]",
- # Print without newlines because it's too hard to indent that to fit under the generated
- # starlark code below.
- args = ["--compact-output"],
-)
+def _oci_alias_impl(rctx):
+ if rctx.attr.platforms and rctx.attr.single_platform:
+ fail("Only one of 'platforms' or 'single_platform' may be set")
+
+ if not rctx.attr.platforms and not rctx.attr.single_platform:
+ fail("One of 'platforms' or 'single_platform' must be set")
+
+ if _is_tag(rctx.attr.identifier) and rctx.attr.reproducible:
+ manifest, _ = _download_manifest(rctx, rctx.attr.identifier, "mf.json")
+ coreutils = _coreutils_label(rctx)
+ result = rctx.execute([coreutils, "hashsum", "--sha256", "mf.json"])
+ if result.return_code:
+ msg = "hashsum failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)
+ fail(msg)
+
+ optional_platforms = ""
+
+ if _is_image_index(manifest):
+ platforms = []
+ for submanifest in manifest["manifests"]:
+ parts = [submanifest["platform"]["os"], submanifest["platform"]["architecture"]]
+ if "variant" in submanifest["platform"]:
+ parts.append(submanifest["platform"]["variant"])
+ platforms.append('"{}"'.format("/".join(parts)))
+ optional_platforms = "'add platforms {}'".format(" ".join(platforms))
 
-jq(
- name = "mediaType",
- srcs = ["manifest_list.json"],
- filter = ".mediaType",
- args = ["--raw-output"],
-)
+ # buildifier: disable=print
+ print("""
+WARNING: for reproducible builds, a digest is recommended.
+Either set 'reproducible = False' to silence this warning,
+or run the following command to change oci_pull to use a digest:
 
-sh_binary(
- name = "pin",
- srcs = ["pin.sh"],
- data = [
- ":mediaType",
- ":platforms",
- "@bazel_tools//tools/bash/runfiles",
- ],
-)
-"""
+buildozer 'set digest "sha256:{digest}"' 'remove tag' 'remove platforms' {optional_platforms} WORKSPACE:{name}
+ """.format(
+ name = rctx.attr.name,
+ digest = result.stdout.split(" ", 1)[0],
+ optional_platforms = optional_platforms,
+ ))
 
-_pin_sh = """\
-#!/usr/bin/env bash
-{rlocation}
+ build = ""
 
-mediaType="$(cat $(rlocation {name}/mediaType.json))"
-echo -e "Replace your '{reponame}' declaration with the following:\n"
+ if rctx.attr.platforms:
+ build = _MULTI_PLATFORM_IMAGE_ALIAS.format(
+ name = rctx.attr.name,
+ platform_map = {str(k): v for k, v in rctx.attr.platforms.items()},
+ )
+ else:
+ build = _SINGLE_PLATFORM_IMAGE_ALIAS.format(
+ name = rctx.attr.name,
+ original = rctx.attr.single_platform.name,
+ )
 
-cat <<EOF
-oci_pull(
- name = "{reponame}",
- digest = "sha256:{digest}",
- image = "{image}",
-EOF
-
-[[ $mediaType == "{manifestListType}" ]] && cat <<EOF
- # Listing of all platforms that were found in the image manifest.
- # You may remove any that you don't use.
- platforms = $(cat $(rlocation {name}/platforms.json)),
-EOF
-
-echo ")"
-"""
+ rctx.file("BUILD.bazel", content = build)
 
-def _pin_tag_impl(rctx):
- """Download the tag and create a repository that can produce pinning instructions"""
- _download_manifest(rctx, rctx.attr.tag, "manifest_list.json")
- result = rctx.execute(["shasum", "-a", "256", "manifest_list.json"])
- if result.return_code:
- msg = "shasum failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)
- fail(msg)
- rctx.file("pin.sh", _pin_sh.format(
- name = rctx.attr.name,
- reponame = rctx.attr.name.replace("_unpinned", ""),
- digest = result.stdout.split(" ", 1)[0],
- image = rctx.attr.image,
- rlocation = BASH_RLOCATION_FUNCTION,
- manifestListType = _MANIFEST_LIST_TYPE,
- ), executable = True)
- rctx.file("BUILD.bazel", _latest_build)
-
-pin_tag = repository_rule(
- _pin_tag_impl,
+oci_alias = repository_rule(
+ implementation = _oci_alias_impl,
  attrs = {
- "image": attr.string(doc = "The name of the image we are fetching, e.g. `gcr.io/distroless/static`", mandatory = True),
- "tag": attr.string(doc = "The tag being used, e.g. `latest`", mandatory = True),
+ "platforms": attr.label_keyed_string_dict(),
+ "single_platform": attr.label(),
+ "identifier": attr.string(),
+ "image": attr.string(doc = "The name of the image we are fetching, e.g. gcr.io/distroless/static", mandatory = True),
  "toolchain_name": attr.string(default = "oci", doc = "Value of name attribute to the oci_register_toolchains call in the workspace."),
+ "reproducible": attr.bool(default = True, doc = "Set to False to silence the warning about reproducibility when using `tag`"),
  },
 )
 
@@ -547,41 +558,36 @@ def oci_pull(name, image, platforms = None, digest = None, tag = None, reproduci
  if not digest and not tag:
  fail("One of 'digest' or 'tag' must be set")
 
- if tag and reproducible:
- pin_tag(name = name + "_unpinned", image = image, tag = tag, toolchain_name = toolchain_name)
-
- # Print a command - in the future we should print a buildozer command or
- # buildifier: disable=print
- print("""
-WARNING: for reproducible builds, a digest is recommended.
-Either set 'reproducible = False' to silence this warning,
-or run the following command to change oci_pull to use a digest:
-
-bazel run @{}_unpinned//:pin
-""".format(name))
- return
+ platform_to_image = None
+ single_platform = None
 
  if platforms:
- select_map = {}
- for plat in platforms:
- plat_name = "_".join([name] + plat.split("/"))
- os, arch = plat.split("/", 1)
+ platform_to_image = {}
+ for platform in platforms:
+ platform_name = "_".join([name] + platform.split("/"))
+ _, arch = platform.split("/", 1)
  oci_pull_rule(
- name = plat_name,
+ name = platform_name,
  image = image,
  identifier = digest or tag,
- platform = plat,
- toolchain_name = toolchain_name,
+ platform = platform,
  )
- select_map[_DOCKER_ARCH_TO_BAZEL_CPU[arch]] = "@" + plat_name
- oci_alias(
- name = name,
- platforms = select_map,
- )
+ platform_to_image[_DOCKER_ARCH_TO_BAZEL_CPU[arch]] = "@" + platform_name
  else:
+ single_platform = "{}_single".format(name)
  oci_pull_rule(
- name = name,
+ name = single_platform,
  image = image,
  identifier = digest or tag,
  toolchain_name = toolchain_name,
  )
+
+ oci_alias(
+ name = name,
+ platforms = platform_to_image,
+ single_platform = single_platform,
+ identifier = digest or tag,
+ image = image,
+ reproducible = reproducible,
+ toolchain_name = toolchain_name,
+ )