From c9cb67fa3811847deba134a78a0fda605c10302d Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Thu, 8 Oct 2020 15:20:35 +0000 Subject: [PATCH] Add multi-arch image via Bazel --- .gitignore | 3 + .travis.yml | 24 +++++ BUILD | 4 + WORKSPACE | 53 ++++++++++ cmd/executor/BUILD | 126 +++++++++++++++++++++++ cmd/executor/cmd/root.go | 6 +- cmd/executor/testdata/Dockerfile.trivial | 2 + cmd/executor/testdata/build.yaml | 5 + cmd/executor/testdata/exec.yaml | 10 ++ cmd/executor/testdata/files.yaml | 12 +++ cmd/executor/testdata/testfiles.yaml | 6 ++ deploy/cloudbuild.yaml | 60 +++++++++++ files/BUILD | 16 +++ 13 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 BUILD create mode 100644 WORKSPACE create mode 100644 cmd/executor/BUILD create mode 100644 cmd/executor/testdata/Dockerfile.trivial create mode 100644 cmd/executor/testdata/build.yaml create mode 100644 cmd/executor/testdata/exec.yaml create mode 100644 cmd/executor/testdata/files.yaml create mode 100644 cmd/executor/testdata/testfiles.yaml create mode 100644 files/BUILD diff --git a/.gitignore b/.gitignore index 6dee7552a6..93bbd96f26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ out/ +bazel-* +*~ +BUILD.bazel .idea *.iml .vagrant diff --git a/.travis.yml b/.travis.yml index 364f2b7c68..bb2d1d9e0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: go os: linux dist: bionic + env: global: - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000 @@ -35,3 +36,26 @@ jobs: - make travis-setup script: - make integration-test-misc + + - arch: amd64 + env: CPU=k8 + before_install: + - export PATH=$PATH:$HOME/bin && mkdir -p $HOME/bin + - eval $(go env) + # install bazelisk as bazel to install the appropriate bazel version + - wget https://github.com/bazelbuild/bazelisk/releases/download/v1.6.1/bazelisk-linux-${GOARCH} && chmod +x bazelisk-linux-${GOARCH} && mv bazelisk-linux-${GOARCH} $HOME/bin/bazel + script: + # Generate BUILD.bazel files (we do not check them in) + - bazel run //:gazelle + - bazel build --cpu=${CPU} --curses=no //... + # Build all targets tagged with our architecture: + - bazel build --cpu=${CPU} --curses=no $(bazel query 'attr("tags", "'${GOARCH}'", "//...")') + # Run all tests not tagged as "manual": + - bazel test --cpu=${CPU} --curses=no --test_output=errors --test_timeout=900 //... + # Run all tests tagged with our architecture: + - bazel test --cpu=${CPU} --curses=no --test_output=errors --test_timeout=900 $(bazel query 'attr("tags", "'${GOARCH}'", "//...")') + +# - arch: arm64 +# env: CPU=aarch64 +# before_install: *before_install_multiarch +# script: *script_multiarch diff --git a/BUILD b/BUILD new file mode 100644 index 0000000000..83716bc4f8 --- /dev/null +++ b/BUILD @@ -0,0 +1,4 @@ +load("@bazel_gazelle//:def.bzl", "gazelle") + +# gazelle:prefix github.com/GoogleContainerTools/kaniko +gazelle(name = "gazelle") diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000000..6300cbfa75 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,53 @@ +workspace(name = "kaniko") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "io_bazel_rules_go", + sha256 = "b725e6497741d7fc2d55fcc29a276627d10e43fa5d0bb692692890ae30d98d00", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz", + ], +) + +http_archive( + name = "bazel_gazelle", + sha256 = "b85f48fa105c4403326e9525ad2b2cc437babaa6e15a3fc0b1dbab0ab064bc7c", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz", + ], +) + +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") + +go_rules_dependencies() + +go_register_toolchains() + +gazelle_dependencies() + +# Docker rules. +http_archive( + name = "io_bazel_rules_docker", + sha256 = "cf53839c398e464b10ec2fbeb11aedb446f078c28e3b4ce372461bb105ef435c", + strip_prefix = "rules_docker-f8478e57ab7457e403fda474f06ac0bb120d92a7", + urls = ["https://github.com/bazelbuild/rules_docker/archive/f8478e57ab7457e403fda474f06ac0bb120d92a7.tar.gz"], +) + +load( + "@io_bazel_rules_docker//repositories:repositories.bzl", + container_repositories = "repositories", +) + +container_repositories() + +load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") + +container_deps() + +load("@io_bazel_rules_docker//repositories:pip_repositories.bzl", "pip_deps") + +pip_deps() diff --git a/cmd/executor/BUILD b/cmd/executor/BUILD new file mode 100644 index 0000000000..19617cd2e2 --- /dev/null +++ b/cmd/executor/BUILD @@ -0,0 +1,126 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//container:container.bzl", "container_image") + +go_library( + name = "executor_lib", + srcs = ["main.go"], + importpath = "github.com/GoogleContainerTools/kaniko/cmd/executor", + visibility = ["//visibility:private"], + deps = ["//cmd/executor/cmd"], +) + +go_binary( + name = "executor", + embed = [":executor_lib"], + pure = "on", + visibility = ["//visibility:public"], +) + +ARCHITECTURES = [ + "amd64", + "arm64", +] + +[ + go_binary( + name = "executor_" + arch, + embed = [":executor_lib"], + goarch = arch, + goos = "linux", + pure = "on", + visibility = ["//visibility:public"], + ) + for arch in ARCHITECTURES +] + +[ + container_image( + name = "image_" + arch, + architecture = arch, + base = "//files:image", + directory = "/kaniko", + entrypoint = ["/kaniko/executor_" + arch], + env = { + "HOME": "/root", + "USER": "root", + "PATH": "/usr/local/bin:/kaniko", + "SSL_CERT_DIR": "/kaniko/ssl/certs", + "DOCKER_CONFIG": "/kaniko/.docker/", + }, + files = [ + ":executor_" + arch, + ], + symlinks = { + "/kaniko/executor": "/kaniko/executor_" + arch, + }, + workdir = "/workspace", + ) + for arch in ARCHITECTURES +] + +# Image with testdata +[ + container_image( + name = "buildtest_image_" + arch, + architecture = arch, + base = ":image_" + arch, + directory = "/workspace", + files = [ + ":testdata/Dockerfile.trivial", + ], + ) + for arch in ARCHITECTURES +] + +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") + +# Non-executable tests can run on any architecture, +# so do not tag them. +[ + container_test( + name = "image_files_" + arch + "_test", + configs = ["testdata/files.yaml"], + image = ":image_" + arch, + # tags = ["manual", arch], + ) + for arch in ARCHITECTURES +] + +[ + container_test( + name = "buildtest_image_" + arch + "_test", + configs = [ + "testdata/files.yaml", + "testdata/testfiles.yaml", + ], + image = ":buildtest_image_" + arch, + # tags = ["manual", arch], + ) + for arch in ARCHITECTURES +] + +[ + container_test( + name = "image_exec_" + arch + "_test", + configs = ["testdata/exec.yaml"], + image = ":image_" + arch, + tags = [ + "manual", + arch, + ], + ) + for arch in ARCHITECTURES +] + +[ + container_test( + name = "image_build_" + arch + "_test", + configs = ["testdata/build.yaml"], + image = ":buildtest_image_" + arch, + tags = [ + "manual", + arch, + ], + ) + for arch in ARCHITECTURES +] diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 983c78169c..718fd2522f 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -95,8 +95,10 @@ var RootCmd = &cobra.Command{ } logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system") } - if err := executor.CheckPushPermissions(opts); err != nil { - exit(errors.Wrap(err, "error checking push permissions -- make sure you entered the correct tag name, and that you are authenticated correctly, and try again")) + if !opts.NoPush { + if err := executor.CheckPushPermissions(opts); err != nil { + exit(errors.Wrap(err, "error checking push permissions -- make sure you entered the correct tag name, and that you are authenticated correctly, and try again")) + } } if err := resolveRelativePaths(); err != nil { exit(errors.Wrap(err, "error resolving relative paths to absolute paths")) diff --git a/cmd/executor/testdata/Dockerfile.trivial b/cmd/executor/testdata/Dockerfile.trivial new file mode 100644 index 0000000000..4fb6c7803b --- /dev/null +++ b/cmd/executor/testdata/Dockerfile.trivial @@ -0,0 +1,2 @@ +FROM ubuntu +RUN echo Hello \ No newline at end of file diff --git a/cmd/executor/testdata/build.yaml b/cmd/executor/testdata/build.yaml new file mode 100644 index 0000000000..b036b2fd8d --- /dev/null +++ b/cmd/executor/testdata/build.yaml @@ -0,0 +1,5 @@ +schemaVersion: "1.0.0" +commandTests: +- name: Trivial build + command: ["/kaniko/executor", "--no-push", "--dockerfile", "./Dockerfile.trivial"] + exitCode: 0 diff --git a/cmd/executor/testdata/exec.yaml b/cmd/executor/testdata/exec.yaml new file mode 100644 index 0000000000..e60382af2a --- /dev/null +++ b/cmd/executor/testdata/exec.yaml @@ -0,0 +1,10 @@ +schemaVersion: "1.0.0" +commandTests: +- name: check fixed name + command: ["/kaniko/executor"] + expectedError: ['Usage:'] + exitCode: 1 +- name: check PATH + command: ["executor"] + expectedError: ['Usage:'] + exitCode: 1 diff --git a/cmd/executor/testdata/files.yaml b/cmd/executor/testdata/files.yaml new file mode 100644 index 0000000000..8f19b259de --- /dev/null +++ b/cmd/executor/testdata/files.yaml @@ -0,0 +1,12 @@ +schemaVersion: "1.0.0" +fileExistenceTests: +# Basic FS sanity checks. +- name: root + path: '/' + shouldExist: true +- name: certs + path: '/kaniko/ssl/certs/ca-certificates.crt' + shouldExist: true +- name: certs + path: '/etc/nsswitch.conf' + shouldExist: true diff --git a/cmd/executor/testdata/testfiles.yaml b/cmd/executor/testdata/testfiles.yaml new file mode 100644 index 0000000000..2a3c4b08b6 --- /dev/null +++ b/cmd/executor/testdata/testfiles.yaml @@ -0,0 +1,6 @@ +schemaVersion: "1.0.0" +fileExistenceTests: +# Basic FS sanity checks. +- name: trivial testdata Dockerfile + path: '/workspace/Dockerfile.trivial' + shouldExist: true diff --git a/deploy/cloudbuild.yaml b/deploy/cloudbuild.yaml index 47a9a0ab20..145155c541 100644 --- a/deploy/cloudbuild.yaml +++ b/deploy/cloudbuild.yaml @@ -1,4 +1,5 @@ steps: + # First, build kaniko - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile", @@ -14,9 +15,68 @@ steps: - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_warmer", "-t", "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}", "."] + + + # Build each of the multi-arch images with Bazel and load them into the Docker daemon. + - name: gcr.io/cloud-marketplace-containers/google/bazel:3.4.1 + entrypoint: sh + args: + - -c + - | + #!/bin/sh + set -o errexit + set -o xtrace + + bazel run //:gazelle + bazel run --host_force_python=PY2 //cmd/executor:image_amd64 + bazel run --host_force_python=PY2 //cmd/executor:image_arm64 + + # Publish the individual container images + - name: docker + entrypoint: sh + args: + - -c + - | + #!/bin/sh + set -o errexit + set -o xtrace + + docker tag bazel/cmd/executor:image_amd64 gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64 + docker tag bazel/cmd/executor:image_arm64 gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64 + + docker push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64 + docker push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64 + + # Enable "manifest list" support in docker, and publish one covering the per-architecture + # images published above. + - name: docker + entrypoint: sh + args: + - -c + - | + #!/bin/sh + set -o errexit + set -o xtrace + + # Publish manifest lists second, after all of the binary material + # has been uploaded, so that it is fast. We want fast because enabling + # the experimental features in docker changes ~/.docker/config.json, which + # GCB periodically tramples. + # + # Enable support for 'docker manifest create' + # https://docs.docker.com/engine/reference/commandline/manifest_create/ + sed -i 's/^{/{"experimental": "enabled",/g' ~/.docker/config.json + + docker manifest create gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:multi-arch \ + gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64 \ + gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64 + docker manifest push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:multi-arch + + images: ["gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}", "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}"] + substitutions: _EXECUTOR_IMAGE_NAME: executor _WARMER_IMAGE_NAME: warmer diff --git a/files/BUILD b/files/BUILD new file mode 100644 index 0000000000..b14343c978 --- /dev/null +++ b/files/BUILD @@ -0,0 +1,16 @@ +load("@io_bazel_rules_docker//container:container.bzl", "container_image") + +container_image( + name = "nsswitch", + directory = "etc", + files = [":nsswitch.conf"], + visibility = ["//visibility:private"], +) + +container_image( + name = "image", + base = ":nsswitch", + directory = "kaniko/ssl/certs", + files = [":ca-certificates.crt"], + visibility = ["//visibility:public"], +)