From ce609b148e31cd2f07b3236e1c5bad16573cff3a Mon Sep 17 00:00:00 2001 From: Michael Plump Date: Fri, 14 Feb 2020 13:07:45 -0500 Subject: [PATCH 1/3] refactor(containers): container builds use GCS for cache storage This makes a few changes: 1. Before the compilation, we pull down a $repo-$branch.tgz file and uncompress it (e.g. igor-master.tgz or rosco-release-1.17.x.tgz) to the /workspace volume 2. The Dockerfile.compile image is now expected to be a build step. This means we can run the container and it will compile files on the /workspace volume. Currently, we copy all the files into the container, compile them there, and then have to copy them back out. This new method is more along the lines with how GCB is supposed to work, but it does mean every Dockerfile.compile has to be changed to use CMD instead of RUN. 3. After the compile, we tar up .gradle/caches and .gradle/wrapper to $repo-$branch.tgz and send it to gs://spinnaker-build-cache. This storage bucket is set to auto-delete files that haven't been modified in 14 days. Removing our dependence on the `gradle_cache` image means that the various microservices can define their own compile image with different build tool configurations. I'd really like to have this for the Java 11 migration. It also means we could move the Debian builds into Cloud Build, since we aren't depending on them to produce that `gradle_cache` artifact. --- dev/buildtool/cloudbuild/README.md | 16 +++++++++ dev/buildtool/cloudbuild/build-steps.yml | 12 +++++++ .../containers-build-java8.yml} | 33 ++++++++++++------- .../containers-tag-java8.yml} | 29 +++++++++++----- dev/buildtool/container_commands.py | 12 ++++--- dev/buildtool/flow_build.sh | 2 +- 6 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 dev/buildtool/cloudbuild/README.md create mode 100644 dev/buildtool/cloudbuild/build-steps.yml rename dev/buildtool/{cloudbuild-build-java8.yml => cloudbuild/containers-build-java8.yml} (68%) rename dev/buildtool/{cloudbuild-tag-java8.yml => cloudbuild/containers-tag-java8.yml} (64%) diff --git a/dev/buildtool/cloudbuild/README.md b/dev/buildtool/cloudbuild/README.md new file mode 100644 index 00000000..42cc7651 --- /dev/null +++ b/dev/buildtool/cloudbuild/README.md @@ -0,0 +1,16 @@ +## Google Cloud Build files + +The `containers-build-java8` and `containers-tag-java8` files are +[Google Cloud Build build configurations](https://cloud.google.com/cloud-build/docs/build-config) +for building the Spinnaker microservices. + +In order to use them, there must be a `save_cache` and `restore_cache` image in +the Google Container Registry of the project in which the configurations are +executed. You can create those images with the `build-steps.yml` file: + +``` +gcloud builds submit --config=build-steps.yml --project=spinnaker-community . +``` + +The source for these images is the +[cloud-builders-community repository](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache). diff --git a/dev/buildtool/cloudbuild/build-steps.yml b/dev/buildtool/cloudbuild/build-steps.yml new file mode 100644 index 00000000..4f708cbe --- /dev/null +++ b/dev/buildtool/cloudbuild/build-steps.yml @@ -0,0 +1,12 @@ +# This file publishes the images that are used by containers-*.yml. It should be +# run manually to create these images. +steps: + - id: fetchCachingRepo + waitFor: ["-"] + name: gcr.io/cloud-builders/git + args: ["clone", "https://github.com/GoogleCloudPlatform/cloud-builders-community"] + # The caching repo contains its own cloud build file which will publish 'save_cache' and 'restore_cache' images. + - id: buildCachingRepo + waitFor: ["fetchCachingRepo"] + name: gcr.io/cloud-builders/gcloud + args: ["builds", "submit", "--config", "cloud-builders-community/cache/cloudbuild.yaml", "cloud-builders-community/cache"] diff --git a/dev/buildtool/cloudbuild-build-java8.yml b/dev/buildtool/cloudbuild/containers-build-java8.yml similarity index 68% rename from dev/buildtool/cloudbuild-build-java8.yml rename to dev/buildtool/cloudbuild/containers-build-java8.yml index ab0ab837..2a458c91 100644 --- a/dev/buildtool/cloudbuild-build-java8.yml +++ b/dev/buildtool/cloudbuild/containers-build-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: $_DOCKER_REGISTRY/restore_cache:latest + args: + - "--bucket=gs://spinnaker-build-cache" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: $_DOCKER_REGISTRY/save_cache:latest + args: + - "--bucket=gs://spinnaker-build-cache/" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -19,7 +30,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -28,7 +39,7 @@ steps: ".", ] - id: buildJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -37,7 +48,7 @@ steps: ".", ] - id: buildUbuntuJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", diff --git a/dev/buildtool/cloudbuild-tag-java8.yml b/dev/buildtool/cloudbuild/containers-tag-java8.yml similarity index 64% rename from dev/buildtool/cloudbuild-tag-java8.yml rename to dev/buildtool/cloudbuild/containers-tag-java8.yml index b7d43067..b8a787b8 100644 --- a/dev/buildtool/cloudbuild-tag-java8.yml +++ b/dev/buildtool/cloudbuild/containers-tag-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: gcr.io/$PROJECT_ID/restore_cache:latest + args: + - "--bucket=gs://spinnaker-build-cache" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: $_DOCKER_REGISTRY/save_cache:latest + args: + - "--bucket=gs://spinnaker-build-cache/" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -20,7 +31,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", diff --git a/dev/buildtool/container_commands.py b/dev/buildtool/container_commands.py index 019db469..264b9f3b 100644 --- a/dev/buildtool/container_commands.py +++ b/dev/buildtool/container_commands.py @@ -93,22 +93,25 @@ def __build_with_gcb(self, repository, build_version): if os.path.isdir(gradle_cache): shutil.rmtree(gradle_cache) - cloudbuild_file_name = 'cloudbuild-tag-java8.yml' + cloudbuild_file_name = 'containers-tag-java8.yml' if os.path.exists(os.path.join(git_dir, 'Dockerfile.java8')): - cloudbuild_file_name = 'cloudbuild-build-java8.yml' + cloudbuild_file_name = 'containers-build-java8.yml' - cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), cloudbuild_file_name) + cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'cloudbuild', + cloudbuild_file_name) service_name = self.scm.repository_name_to_service_name(repository.name) # Note this command assumes a cwd of git_dir command = ('gcloud builds submit ' ' --account={account} --project={project}' - ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry}' + ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry},_BRANCH_NAME={branch_name},' ' --config={cloudbuild_config} .' .format(account=options.gcb_service_account, project=options.gcb_project, docker_registry=options.docker_registry, tag_name=build_version, image_name=service_name, + branch_name=options.git_branch, cloudbuild_config=cloudbuild_config)) logfile = self.get_logfile_path(name + '-gcb-build') @@ -135,6 +138,7 @@ def init_argparser(self, parser, defaults): super(BuildContainerFactory, self).init_argparser(parser, defaults) self.add_bom_parser_args(parser, defaults) + BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'gcb_project', defaults, None, help='The GCP project ID that builds the containers when' diff --git a/dev/buildtool/flow_build.sh b/dev/buildtool/flow_build.sh index 50542066..ad25d2da 100755 --- a/dev/buildtool/flow_build.sh +++ b/dev/buildtool/flow_build.sh @@ -43,7 +43,7 @@ function run_build_flow() { wait_for_commands_or_die "Bom" start_command_unless NO_CONTAINERS "build_bom_containers" \ - $EXTRA_BOM_COMMAND_ARGS + $EXTRA_BOM_COMMAND_ARGS --git_branch $BOM_BRANCH start_command_unless NO_DEBIANS "build_debians" \ $EXTRA_BOM_COMMAND_ARGS \ "--max_local_builds=6" From 562380413f5ccb432253621c25e7eebe6252230b Mon Sep 17 00:00:00 2001 From: Michael Plump Date: Tue, 18 Feb 2020 16:41:05 -0500 Subject: [PATCH 2/3] refactor(containers): make the cache bucket configurable --- dev/buildtool/cloudbuild/containers-build-java8.yml | 6 ++++-- dev/buildtool/cloudbuild/containers-tag-java8.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dev/buildtool/cloudbuild/containers-build-java8.yml b/dev/buildtool/cloudbuild/containers-build-java8.yml index 2a458c91..ddc7fb91 100644 --- a/dev/buildtool/cloudbuild/containers-build-java8.yml +++ b/dev/buildtool/cloudbuild/containers-build-java8.yml @@ -2,7 +2,7 @@ steps: - id: restoreCache name: $_DOCKER_REGISTRY/restore_cache:latest args: - - "--bucket=gs://spinnaker-build-cache" + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage waitFor: ["restoreCache"] @@ -15,7 +15,7 @@ steps: waitFor: ["compile"] name: $_DOCKER_REGISTRY/save_cache:latest args: - - "--bucket=gs://spinnaker-build-cache/" + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - "--path=.gradle/caches" - "--path=.gradle/wrapper" @@ -65,3 +65,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache diff --git a/dev/buildtool/cloudbuild/containers-tag-java8.yml b/dev/buildtool/cloudbuild/containers-tag-java8.yml index b8a787b8..07941ca8 100644 --- a/dev/buildtool/cloudbuild/containers-tag-java8.yml +++ b/dev/buildtool/cloudbuild/containers-tag-java8.yml @@ -2,7 +2,7 @@ steps: - id: restoreCache name: gcr.io/$PROJECT_ID/restore_cache:latest args: - - "--bucket=gs://spinnaker-build-cache" + - "--bucket=gs://_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage waitFor: ["restoreCache"] @@ -15,7 +15,7 @@ steps: waitFor: ["compile"] name: $_DOCKER_REGISTRY/save_cache:latest args: - - "--bucket=gs://spinnaker-build-cache/" + - "--bucket=gs://_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - "--path=.gradle/caches" - "--path=.gradle/wrapper" @@ -49,3 +49,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache From 0481d52828fd8401b3321f47bf6e283b7da146a5 Mon Sep 17 00:00:00 2001 From: Michael Plump Date: Thu, 20 Feb 2020 10:21:03 -0500 Subject: [PATCH 3/3] fix(containers): use {save,restore}_cache from the build project --- dev/buildtool/cloudbuild/containers-build-java8.yml | 4 ++-- dev/buildtool/cloudbuild/containers-tag-java8.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/buildtool/cloudbuild/containers-build-java8.yml b/dev/buildtool/cloudbuild/containers-build-java8.yml index ddc7fb91..55e13bb5 100644 --- a/dev/buildtool/cloudbuild/containers-build-java8.yml +++ b/dev/buildtool/cloudbuild/containers-build-java8.yml @@ -1,6 +1,6 @@ steps: - id: restoreCache - name: $_DOCKER_REGISTRY/restore_cache:latest + name: gcr.io/$PROJECT_ID/restore_cache:latest args: - "--bucket=gs://$_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" @@ -13,7 +13,7 @@ steps: name: compile - id: saveCache waitFor: ["compile"] - name: $_DOCKER_REGISTRY/save_cache:latest + name: gcr.io/$PROJECT_ID/save_cache:latest args: - "--bucket=gs://$_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" diff --git a/dev/buildtool/cloudbuild/containers-tag-java8.yml b/dev/buildtool/cloudbuild/containers-tag-java8.yml index 07941ca8..e8ca7de9 100644 --- a/dev/buildtool/cloudbuild/containers-tag-java8.yml +++ b/dev/buildtool/cloudbuild/containers-tag-java8.yml @@ -2,7 +2,7 @@ steps: - id: restoreCache name: gcr.io/$PROJECT_ID/restore_cache:latest args: - - "--bucket=gs://_COMPILE_CACHE_BUCKET" + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage waitFor: ["restoreCache"] @@ -13,12 +13,12 @@ steps: name: compile - id: saveCache waitFor: ["compile"] - name: $_DOCKER_REGISTRY/save_cache:latest + name: gcr.io/$PROJECT_ID/save_cache:latest args: - - "--bucket=gs://_COMPILE_CACHE_BUCKET" - - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - - "--path=.gradle/caches" - - "--path=.gradle/wrapper" + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" - id: buildSlimImage waitFor: ["compile"] name: gcr.io/cloud-builders/docker