diff --git a/dev/buildtool/cloudbuild/README.md b/dev/buildtool/cloudbuild/README.md new file mode 100644 index 00000000..42cc7651 --- /dev/null +++ b/dev/buildtool/cloudbuild/README.md @@ -0,0 +1,16 @@ +## Google Cloud Build files + +The `containers-build-java8` and `containers-tag-java8` files are +[Google Cloud Build build configurations](https://cloud.google.com/cloud-build/docs/build-config) +for building the Spinnaker microservices. + +In order to use them, there must be a `save_cache` and `restore_cache` image in +the Google Container Registry of the project in which the configurations are +executed. You can create those images with the `build-steps.yml` file: + +``` +gcloud builds submit --config=build-steps.yml --project=spinnaker-community . +``` + +The source for these images is the +[cloud-builders-community repository](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache). diff --git a/dev/buildtool/cloudbuild/build-steps.yml b/dev/buildtool/cloudbuild/build-steps.yml new file mode 100644 index 00000000..4f708cbe --- /dev/null +++ b/dev/buildtool/cloudbuild/build-steps.yml @@ -0,0 +1,12 @@ +# This file publishes the images that are used by containers-*.yml. It should be +# run manually to create these images. +steps: + - id: fetchCachingRepo + waitFor: ["-"] + name: gcr.io/cloud-builders/git + args: ["clone", "https://github.com/GoogleCloudPlatform/cloud-builders-community"] + # The caching repo contains its own cloud build file which will publish 'save_cache' and 'restore_cache' images. + - id: buildCachingRepo + waitFor: ["fetchCachingRepo"] + name: gcr.io/cloud-builders/gcloud + args: ["builds", "submit", "--config", "cloud-builders-community/cache/cloudbuild.yaml", "cloud-builders-community/cache"] diff --git a/dev/buildtool/cloudbuild-build-java8.yml b/dev/buildtool/cloudbuild/containers-build-java8.yml similarity index 66% rename from dev/buildtool/cloudbuild-build-java8.yml rename to dev/buildtool/cloudbuild/containers-build-java8.yml index ab0ab837..55e13bb5 100644 --- a/dev/buildtool/cloudbuild-build-java8.yml +++ b/dev/buildtool/cloudbuild/containers-build-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: gcr.io/$PROJECT_ID/restore_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: gcr.io/$PROJECT_ID/save_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -19,7 +30,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -28,7 +39,7 @@ steps: ".", ] - id: buildJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -37,7 +48,7 @@ steps: ".", ] - id: buildUbuntuJava8Image - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -54,3 +65,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache diff --git a/dev/buildtool/cloudbuild-tag-java8.yml b/dev/buildtool/cloudbuild/containers-tag-java8.yml similarity index 61% rename from dev/buildtool/cloudbuild-tag-java8.yml rename to dev/buildtool/cloudbuild/containers-tag-java8.yml index b7d43067..e8ca7de9 100644 --- a/dev/buildtool/cloudbuild-tag-java8.yml +++ b/dev/buildtool/cloudbuild/containers-tag-java8.yml @@ -1,15 +1,26 @@ steps: + - id: restoreCache + name: gcr.io/$PROJECT_ID/restore_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" - id: buildCompileImage - waitFor: ["-"] + waitFor: ["restoreCache"] name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "compile", - "-f", "Dockerfile.compile", - ".", - ] - - id: buildSlimImage + args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."] + - id: compile waitFor: ["buildCompileImage"] + name: compile + - id: saveCache + waitFor: ["compile"] + name: gcr.io/$PROJECT_ID/save_cache:latest + args: + - "--bucket=gs://$_COMPILE_CACHE_BUCKET" + - "--key=$_IMAGE_NAME-$_BRANCH_NAME" + - "--path=.gradle/caches" + - "--path=.gradle/wrapper" + - id: buildSlimImage + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -20,7 +31,7 @@ steps: ".", ] - id: buildUbuntuImage - waitFor: ["buildCompileImage"] + waitFor: ["compile"] name: gcr.io/cloud-builders/docker args: [ "build", @@ -38,3 +49,5 @@ images: timeout: 3600s options: machineType: N1_HIGHCPU_8 +substitutions: + _COMPILE_CACHE_BUCKET: spinnaker-build-cache diff --git a/dev/buildtool/container_commands.py b/dev/buildtool/container_commands.py index 019db469..264b9f3b 100644 --- a/dev/buildtool/container_commands.py +++ b/dev/buildtool/container_commands.py @@ -93,22 +93,25 @@ def __build_with_gcb(self, repository, build_version): if os.path.isdir(gradle_cache): shutil.rmtree(gradle_cache) - cloudbuild_file_name = 'cloudbuild-tag-java8.yml' + cloudbuild_file_name = 'containers-tag-java8.yml' if os.path.exists(os.path.join(git_dir, 'Dockerfile.java8')): - cloudbuild_file_name = 'cloudbuild-build-java8.yml' + cloudbuild_file_name = 'containers-build-java8.yml' - cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), cloudbuild_file_name) + cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'cloudbuild', + cloudbuild_file_name) service_name = self.scm.repository_name_to_service_name(repository.name) # Note this command assumes a cwd of git_dir command = ('gcloud builds submit ' ' --account={account} --project={project}' - ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry}' + ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry},_BRANCH_NAME={branch_name},' ' --config={cloudbuild_config} .' .format(account=options.gcb_service_account, project=options.gcb_project, docker_registry=options.docker_registry, tag_name=build_version, image_name=service_name, + branch_name=options.git_branch, cloudbuild_config=cloudbuild_config)) logfile = self.get_logfile_path(name + '-gcb-build') @@ -135,6 +138,7 @@ def init_argparser(self, parser, defaults): super(BuildContainerFactory, self).init_argparser(parser, defaults) self.add_bom_parser_args(parser, defaults) + BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'gcb_project', defaults, None, help='The GCP project ID that builds the containers when' diff --git a/dev/buildtool/flow_build.sh b/dev/buildtool/flow_build.sh index 50542066..ad25d2da 100755 --- a/dev/buildtool/flow_build.sh +++ b/dev/buildtool/flow_build.sh @@ -43,7 +43,7 @@ function run_build_flow() { wait_for_commands_or_die "Bom" start_command_unless NO_CONTAINERS "build_bom_containers" \ - $EXTRA_BOM_COMMAND_ARGS + $EXTRA_BOM_COMMAND_ARGS --git_branch $BOM_BRANCH start_command_unless NO_DEBIANS "build_debians" \ $EXTRA_BOM_COMMAND_ARGS \ "--max_local_builds=6"