From e1a739be47ee2c8be77715ac9b3313cc2e93ca15 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Thu, 16 Mar 2023 15:48:57 +0300 Subject: [PATCH] refactor: make pinning instructions a warning mesage --- WORKSPACE | 9 ++ docs/pull.md | 30 ++----- oci/BUILD.bazel | 1 - oci/pull.bzl | 230 +++++++++++++++++++++++++----------------------- 4 files changed, 134 insertions(+), 136 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 5d834045..28cd4fcd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -113,3 +113,12 @@ new_local_repository( build_file = "//examples/attest_external:BUILD.template", path = "examples/attest_external/workspace", ) + +oci_pull( + name = "thesayyn_debian", + image = "index.docker.io/library/debian", + platforms = ["linux/arm64"], + # Don't make a distroless_python_unpinned repo and print a warning about the tag + reproducible = False, + tag = "latest", +) diff --git a/docs/pull.md b/docs/pull.md index 77c67ca3..d4794cf1 100644 --- a/docs/pull.md +++ b/docs/pull.md @@ -43,7 +43,8 @@ oci_image( ## oci_alias 
-oci_alias(name, platforms, repo_mapping)
+oci_alias(name, identifier, image, platforms, repo_mapping, reproducible, single_platform,
+          toolchain_name)
@@ -54,8 +55,13 @@ oci_alias(name, pla | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this repository. | Name | required | | +| identifier | - | String | optional | "" | +| image | The name of the image we are fetching, e.g. gcr.io/distroless/static | String | required | | | platforms | - | Dictionary: Label -> String | optional | {} | | repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | +| reproducible | Set to False to silence the warning about reproducibility when using tag | Boolean | optional | True | +| single_platform | - | Label | optional | None | +| toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" | @@ -81,28 +87,6 @@ oci_pull_rule(name, toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" | - - -## pin_tag - -
-pin_tag(name, image, repo_mapping, tag, toolchain_name)
-
- - - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| image | The name of the image we are fetching, e.g. gcr.io/distroless/static | String | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| tag | The tag being used, e.g. latest | String | required | | -| toolchain_name | Value of name attribute to the oci_register_toolchains call in the workspace. | String | optional | "oci" | - - ## oci_pull diff --git a/oci/BUILD.bazel b/oci/BUILD.bazel index 27d39306..6688b62e 100644 --- a/oci/BUILD.bazel +++ b/oci/BUILD.bazel @@ -41,7 +41,6 @@ bzl_library( visibility = ["//visibility:public"], deps = [ "@aspect_bazel_lib//lib:base64", - "@aspect_bazel_lib//lib:paths", "@aspect_bazel_lib//lib:repo_utils", ], ) diff --git a/oci/pull.bzl b/oci/pull.bzl index 657230d8..67ba0858 100644 --- a/oci/pull.bzl +++ b/oci/pull.bzl @@ -36,7 +36,6 @@ oci_image( ``` """ -load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION") load("@aspect_bazel_lib//lib:base64.bzl", "base64") load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils") @@ -168,8 +167,34 @@ Running one of `podman login`, `docker login`, `crane login` may help. # OCI Image Media Types # Spec: https://github.com/distribution/distribution/blob/main/docs/spec/manifest-v2-2.md#media-types -_MANIFEST_TYPE = "application/vnd.docker.distribution.manifest.v2+json" -_MANIFEST_LIST_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json" +OCI_IMAGE_INDEX = "application/vnd.oci.image.index.v1+json" +DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +OCI_MANIFEST_SCHEMA1 = "application/vnd.oci.image.manifest.v1+json" +DOCKER_MANIFEST_SCHEMA2 = "application/vnd.docker.distribution.manifest.v2+json" + +def _is_image_index(descriptor): + media_type = descriptor["mediaType"] + return media_type == OCI_IMAGE_INDEX or media_type == DOCKER_MANIFEST_LIST + +def _is_image(descriptor): + media_type = descriptor["mediaType"] + return media_type == OCI_MANIFEST_SCHEMA1 or media_type == DOCKER_MANIFEST_SCHEMA2 + +def _parse_reference(reference): + firstslash = reference.find("/") + registry = reference[:firstslash] + repository = reference[firstslash + 1:] + return registry, repository + +def _is_tag(str): + return str.find(":") == -1 + +def _trim_hash_algorithm(identifier): + "Optionally remove the sha256: prefix from identifier, if present" + parts = identifier.split(":", 1) + if len(parts) != 2: + return identifier + return parts[1] def _parse_reference(reference): firstslash = reference.find("/") @@ -314,18 +339,20 @@ def _oci_pull_impl(rctx): mf_file = _trim_hash_algorithm(rctx.attr.identifier) mf, mf_len = _download_manifest(rctx, rctx.attr.identifier, mf_file) - if mf["mediaType"] == _MANIFEST_TYPE: + if _is_image(mf): if rctx.attr.platform: fail("{} is a single-architecture image, so attribute 'platform' should not be set.".format(rctx.attr.image)) image_mf_file = mf_file image_mf = mf image_mf_len = mf_len image_digest = rctx.attr.identifier - elif mf["mediaType"] == _MANIFEST_LIST_TYPE: + elif _is_image_index(mf): # extra download to get the manifest for the selected arch if not rctx.attr.platform: fail("{} is a multi-architecture image, so attribute 'platform' is required.".format(rctx.attr.image)) - matching_mf = _find_platform_manifest(rctx, mf) + matching_mf = _find_platform_manifest(mf, rctx.attr.platform) + if not matching_mf: + fail("No matching manifest found in image {} for platform {}".format(rctx.attr.image, rctx.attr.platform)) image_digest = matching_mf["digest"] image_mf_file = _trim_hash_algorithm(image_digest) image_mf, image_mf_len = _download_manifest(rctx, image_digest, image_mf_file) @@ -407,7 +434,7 @@ oci_pull_rule = repository_rule( ], ) -_alias_target = """\ +_MULTI_PLATFORM_IMAGE_ALIAS = """\ alias( name = "{name}", actual = select( @@ -417,96 +444,80 @@ alias( ) """ -def _oci_alias_impl(rctx): - rctx.file("BUILD.bazel", content = _alias_target.format( - name = rctx.attr.name, - platform_map = {str(k): v for k, v in rctx.attr.platforms.items()}, - )) - -oci_alias = repository_rule( - implementation = _oci_alias_impl, - attrs = { - "platforms": attr.label_keyed_string_dict(), - }, +_SINGLE_PLATFORM_IMAGE_ALIAS = """\ +alias( + name = "{name}", + actual = "@{original}//:{original}", + visibility = ["//visibility:public"], ) +""" -_latest_build = """\ -load("@aspect_bazel_lib//lib:jq.bzl", "jq") -load("@bazel_skylib//rules:write_file.bzl", "write_file") +def _coreutils_label(rctx): + return Label("@coreutils_{}//:coreutils".format(repo_utils.platform(rctx))) -jq( - name = "platforms", - srcs = ["manifest_list.json"], - filter = "[(.manifests // [])[] | .platform | .os + \\"/\\" + .architecture]", - # Print without newlines because it's too hard to indent that to fit under the generated - # starlark code below. - args = ["--compact-output"], -) +def _oci_alias_impl(rctx): + if rctx.attr.platforms and rctx.attr.single_platform: + fail("Only one of 'platforms' or 'single_platform' may be set") + + if not rctx.attr.platforms and not rctx.attr.single_platform: + fail("One of 'platforms' or 'single_platform' must be set") + + if _is_tag(rctx.attr.identifier) and rctx.attr.reproducible: + manifest, _ = _download_manifest(rctx, rctx.attr.identifier, "mf.json") + coreutils = _coreutils_label(rctx) + result = rctx.execute([coreutils, "hashsum", "--sha256", "mf.json"]) + if result.return_code: + msg = "hashsum failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr) + fail(msg) + + optional_platforms = "" + + if _is_image_index(manifest): + platforms = [] + for submanifest in manifest["manifests"]: + parts = [submanifest["platform"]["os"], submanifest["platform"]["architecture"]] + if "variant" in submanifest["platform"]: + parts.append(submanifest["platform"]["variant"]) + platforms.append('"{}"'.format("/".join(parts))) + optional_platforms = "'add platforms {}'".format(" ".join(platforms)) -jq( - name = "mediaType", - srcs = ["manifest_list.json"], - filter = ".mediaType", - args = ["--raw-output"], -) + # buildifier: disable=print + print(""" +WARNING: for reproducible builds, a digest is recommended. +Either set 'reproducible = False' to silence this warning, +or run the following command to change oci_pull to use a digest: -sh_binary( - name = "pin", - srcs = ["pin.sh"], - data = [ - ":mediaType", - ":platforms", - "@bazel_tools//tools/bash/runfiles", - ], -) -""" +buildozer 'set digest "sha256:{digest}"' 'remove tag' 'remove platforms' {optional_platforms} WORKSPACE:{name} + """.format( + name = rctx.attr.name, + digest = result.stdout.split(" ", 1)[0], + optional_platforms = optional_platforms, + )) -_pin_sh = """\ -#!/usr/bin/env bash -{rlocation} + build = "" -mediaType="$(cat $(rlocation {name}/mediaType.json))" -echo -e "Replace your '{reponame}' declaration with the following:\n" + if rctx.attr.platforms: + build = _MULTI_PLATFORM_IMAGE_ALIAS.format( + name = rctx.attr.name, + platform_map = {str(k): v for k, v in rctx.attr.platforms.items()}, + ) + else: + build = _SINGLE_PLATFORM_IMAGE_ALIAS.format( + name = rctx.attr.name, + original = rctx.attr.single_platform.name, + ) -cat <