From e252e0e5b14599fa2d53ddff61f5d72f3ba416f2 Mon Sep 17 00:00:00 2001 From: Michael Plump Date: Mon, 24 Feb 2020 21:30:42 -0500 Subject: [PATCH] refactor(containers): container builds use GCS for cache storage (#14) * refactor(containers): container builds use GCS for cache storage This makes a few changes: 1. Before the compilation, we pull down a $repo-$branch.tgz file and uncompress it (e.g. igor-master.tgz or rosco-release-1.17.x.tgz) to the /workspace volume 2. The Dockerfile.compile image is now expected to be a build step. This means we can run the container and it will compile files on the /workspace volume. Currently, we copy all the files into the container, compile them there, and then have to copy them back out. This new method is more along the lines with how GCB is supposed to work, but it does mean every Dockerfile.compile has to be changed to use CMD instead of RUN. 3. After the compile, we tar up .gradle/caches and .gradle/wrapper to $repo-$branch.tgz and send it to gs://spinnaker-build-cache. This storage bucket is set to auto-delete files that haven't been modified in 14 days. Removing our dependence on the `gradle_cache` image means that the various microservices can define their own compile image with different build tool configurations. I'd really like to have this for the Java 11 migration. It also means we could move the Debian builds into Cloud Build, since we aren't depending on them to produce that `gradle_cache` artifact. * refactor(containers): make the cache bucket configurable * fix(containers): use {save,restore}_cache from the build project --- dev/buildtool/cloudbuild/README.md | 16 +++++++++ dev/buildtool/cloudbuild/build-steps.yml | 12 +++++++ .../containers-build-java8.yml} | 35 +++++++++++++------ .../containers-tag-java8.yml} | 31 +++++++++++----- dev/buildtool/container_commands.py | 12 ++++--- dev/buildtool/flow_build.sh | 2 +- 6 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 dev/buildtool/cloudbuild/README.md create mode 100644 dev/buildtool/cloudbuild/build-steps.yml rename dev/buildtool/{cloudbuild-build-java8.yml => cloudbuild/containers-build-java8.yml} (66%) rename dev/buildtool/{cloudbuild-tag-java8.yml => cloudbuild/containers-tag-java8.yml} (61%) diff --git a/dev/buildtool/cloudbuild/README.md b/dev/buildtool/cloudbuild/README.md new file mode 100644 index 00000000..42cc7651 --- /dev/null +++ b/dev/buildtool/cloudbuild/README.md @@ -0,0 +1,16 @@ +## Google Cloud Build files + +The `containers-build-java8` and `containers-tag-java8` files are +[Google Cloud Build build configurations](https://cloud.google.com/cloud-build/docs/build-config) +for building the Spinnaker microservices. + +In order to use them, there must be a `save_cache` and `restore_cache` image in +the Google Container Registry of the project in which the configurations are +executed. You can create those images with the `build-steps.yml` file: + +``` +gcloud builds submit --config=build-steps.yml --project=spinnaker-community . +``` + +The source for these images is the +[cloud-builders-community repository](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache). diff --git a/dev/buildtool/cloudbuild/build-steps.yml b/dev/buildtool/cloudbuild/build-steps.yml new file mode 100644 index 00000000..4f708cbe --- /dev/null +++ b/dev/buildtool/cloudbuild/build-steps.yml @@ -0,0 +1,12 @@ +# This file publishes the images that are used by containers-*.yml. It should be +# run manually to create these images. +steps: + - id: fetchCachingRepo + waitFor: ["-"] + name: gcr.io/cloud-builders/git + args: ["clone", "https://github.com/GoogleCloudPlatform/cloud-builders-community"] + # The caching repo contains its own cloud build file which will publish 'save_cache' and 'restore_cache' images. + - id: buildCachingRepo + waitFor: ["fetchCachingRepo"] + name: gcr.io/cloud-builders/gcloud + args: ["builds", "submit", "--config", "cloud-builders-community/cache/cloudbuild.yaml", "cloud-builders-community/cache"] diff --git a/dev/buildtool/cloudbuild-build-java8.yml b/dev/buildtool/cloudbuild/containers-build-java8.yml similarity index 66% rename from dev/buildtool/cloudbuild-build-java8.yml rename to dev/buildtool/cloudbuild/containers-build-java8.yml index ab0ab837..55e13bb5 100644 --- a/dev/buildtool/cloudbuild-build-java8.yml +++ b/dev/buildtool/cloudbuild/containers-build-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: gcr.io/$PROJECT_ID/restore_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: gcr.io/$PROJECT_ID/save_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -19,7 +30,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -28,7 +39,7 @@ steps: ".", ] - id: buildJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -37,7 +48,7 @@ steps: ".", ] - id: buildUbuntuJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -54,3 +65,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache diff --git a/dev/buildtool/cloudbuild-tag-java8.yml b/dev/buildtool/cloudbuild/containers-tag-java8.yml similarity index 61% rename from dev/buildtool/cloudbuild-tag-java8.yml rename to dev/buildtool/cloudbuild/containers-tag-java8.yml index b7d43067..e8ca7de9 100644 --- a/dev/buildtool/cloudbuild-tag-java8.yml +++ b/dev/buildtool/cloudbuild/containers-tag-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: gcr.io/$PROJECT_ID/restore_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: gcr.io/$PROJECT_ID/save_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -20,7 +31,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -38,3 +49,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache diff --git a/dev/buildtool/container_commands.py b/dev/buildtool/container_commands.py index 019db469..264b9f3b 100644 --- a/dev/buildtool/container_commands.py +++ b/dev/buildtool/container_commands.py @@ -93,22 +93,25 @@ def __build_with_gcb(self, repository, build_version): if os.path.isdir(gradle_cache): shutil.rmtree(gradle_cache) - cloudbuild_file_name = 'cloudbuild-tag-java8.yml' + cloudbuild_file_name = 'containers-tag-java8.yml' if os.path.exists(os.path.join(git_dir, 'Dockerfile.java8')): - cloudbuild_file_name = 'cloudbuild-build-java8.yml' + cloudbuild_file_name = 'containers-build-java8.yml' - cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), cloudbuild_file_name) + cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'cloudbuild', + cloudbuild_file_name) service_name = self.scm.repository_name_to_service_name(repository.name) # Note this command assumes a cwd of git_dir command = ('gcloud builds submit ' ' --account={account} --project={project}' - ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry}' + ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry},_BRANCH_NAME={branch_name},' ' --config={cloudbuild_config} .' .format(account=options.gcb_service_account, project=options.gcb_project, docker_registry=options.docker_registry, tag_name=build_version, image_name=service_name, + branch_name=options.git_branch, cloudbuild_config=cloudbuild_config)) logfile = self.get_logfile_path(name + '-gcb-build') @@ -135,6 +138,7 @@ def init_argparser(self, parser, defaults): super(BuildContainerFactory, self).init_argparser(parser, defaults) self.add_bom_parser_args(parser, defaults) + BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'gcb_project', defaults, None, help='The GCP project ID that builds the containers when' diff --git a/dev/buildtool/flow_build.sh b/dev/buildtool/flow_build.sh index 50542066..ad25d2da 100755 --- a/dev/buildtool/flow_build.sh +++ b/dev/buildtool/flow_build.sh @@ -43,7 +43,7 @@ function run_build_flow() { wait_for_commands_or_die "Bom" start_command_unless NO_CONTAINERS "build_bom_containers" \ - $EXTRA_BOM_COMMAND_ARGS + $EXTRA_BOM_COMMAND_ARGS --git_branch $BOM_BRANCH start_command_unless NO_DEBIANS "build_debians" \ $EXTRA_BOM_COMMAND_ARGS \ "--max_local_builds=6"