From 3fa7cf7617783301b9bfe4011f7ec1f15e216fd5 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Fri, 12 Aug 2022 18:14:21 -0700 Subject: [PATCH 01/21] replace parser map var with individual parser vars --- terraform/modules/fourkeys/variables.tf | 46 ++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/terraform/modules/fourkeys/variables.tf b/terraform/modules/fourkeys/variables.tf index 03a0564e..bde37476 100644 --- a/terraform/modules/fourkeys/variables.tf +++ b/terraform/modules/fourkeys/variables.tf @@ -33,23 +33,53 @@ variable "enable_apis" { variable "enable_build_images" { type = bool description = "Toggle to build fourkeys images and upload to container registry. If set to false, URLs for images must be provided via the container_url variables" - default = true + default = false } variable "event_handler_container_url" { type = string - description = "If 'enable_build_images' is set to false, this is the URL for the event_handler container image." + description = "The URL for the event_handler container image. A default value pointing to the project's container registry is defined in under local values of this module." default = "" } variable "dashboard_container_url" { type = string - description = "If 'enable_build_images' is set to false, this is the URL for the dashboard container image." + description = "The URL for the dashboard container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} + +variable "github_parser_url" { + type = string + description = "The URL for the Github parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} + +variable "gitlab_parser_url" { + type = string + description = "The URL for the Gitlab parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} + +variable "cloud_build_parser_url" { + type = string + description = "The URL for the Cloud Build parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} + +variable "tekton_parser_url" { + type = string + description = "The URL for the Tekton parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} + +variable "circleci_parser_url" { + type = string + description = "The URL for the CircleCI parser container image. A default value pointing to the project's container registry is defined in under local values of this module." default = "" } -variable "parser_container_urls" { - type = map(any) - description = "If 'enable_build_images' is set to false, this is the URL for the parser container images. e.g: {'github': 'gcr.io/youproject/github-parser', 'gitlab': 'gcr.io/youproject/gitlab-parser'} " - default = {} -} \ No newline at end of file +variable "pagerduty_parser_url" { + type = string + description = "The URL for the Pager Duty parser container image. A default value pointing to the project's container registry is defined in under local values of this module." + default = "" +} From 6eeb17aa5c49556ef7881b2334b4d4a40e53a3d2 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Fri, 12 Aug 2022 18:14:53 -0700 Subject: [PATCH 02/21] refactor locals to use new vars --- terraform/modules/fourkeys/locals.tf | 18 ++++++++---------- terraform/modules/fourkeys/parsers.tf | 12 ++++++------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/terraform/modules/fourkeys/locals.tf b/terraform/modules/fourkeys/locals.tf index 5546dbce..a8f92647 100644 --- a/terraform/modules/fourkeys/locals.tf +++ b/terraform/modules/fourkeys/locals.tf @@ -4,16 +4,14 @@ data "google_project" "project" { locals { cloud_build_service_account = "${data.google_project.project.number}@cloudbuild.gserviceaccount.com" - event_handler_container_url = var.enable_build_images ? format("gcr.io/%s/event-handler", var.project_id) : var.event_handler_container_url - dashboard_container_url = var.enable_build_images ? format("gcr.io/%s/fourkeys-grafana-dashboard", var.project_id) : var.dashboard_container_url - parser_container_urls = var.enable_build_images ? { - "github" = format("gcr.io/%s/github-parser", var.project_id) - "gitlab" = format("gcr.io/%s/gitlab-parser", var.project_id) - "cloud-build" = format("gcr.io/%s/cloud-build-parser", var.project_id) - "tekton" = format("gcr.io/%s/tekton-parser", var.project_id) - "circleci" = format("gcr.io/%s/circleci-parser", var.project_id) - "pagerduty" = format("gcr.io/%s/pagerduty-parser", var.project_id) - } : var.parser_container_urls + event_handler_container_url = var.event_handler_container_url == "" ? format("gcr.io/%s/event-handler", var.project_id) : var.event_handler_container_url + dashboard_container_url = var.dashboard_container_url == "" ? format("gcr.io/%s/fourkeys-grafana-dashboard", var.project_id) : var.dashboard_container_url + github_parser_url = var.github_parser_url == "" ? format("gcr.io/%s/github-parser", var.project_id) : var.github_parser_url + gitlab_parser_url = var.gitlab_parser_url == "" ? format("gcr.io/%s/gitlab-parser", var.project_id) : var.gitlab_parser_url + cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("gcr.io/%s/cloud-build-parser", var.project_id) : var.cloud_build_parser_url + tekton_parser_url = var.tekton_parser_url == "" ? format("gcr.io/%s/tekton-parser", var.project_id) : var.tekton_parser_url + circleci_parser_url = var.circleci_parser_url == "" ? format("gcr.io/%s/circleci-parser", var.project_id) : var.circleci_parser_url + pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("gcr.io/%s/pagerduty-parser", var.project_id) : var.pagerduty_parser_url services = var.enable_apis ? [ "bigquery.googleapis.com", "cloudbuild.googleapis.com", diff --git a/terraform/modules/fourkeys/parsers.tf b/terraform/modules/fourkeys/parsers.tf index 19218afb..65b31c57 100644 --- a/terraform/modules/fourkeys/parsers.tf +++ b/terraform/modules/fourkeys/parsers.tf @@ -2,7 +2,7 @@ module "circleci_parser" { source = "../fourkeys-circleci-parser" count = contains(var.parsers, "circleci") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["circleci"] + parser_container_url = local.circleci_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis @@ -15,7 +15,7 @@ module "github_parser" { source = "../fourkeys-github-parser" count = contains(var.parsers, "github") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["github"] + parser_container_url = local.github_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis @@ -28,7 +28,7 @@ module "gitlab_parser" { source = "../fourkeys-gitlab-parser" count = contains(var.parsers, "gitlab") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["gitlab"] + parser_container_url = local.gitlab_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis @@ -41,7 +41,7 @@ module "pagerduty_parser" { source = "../fourkeys-pagerduty-parser" count = contains(var.parsers, "pagerduty") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["pagerduty"] + parser_container_url = local.pagerduty_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis @@ -54,7 +54,7 @@ module "tekton_parser" { source = "../fourkeys-tekton-parser" count = contains(var.parsers, "tekton") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["tekton"] + parser_container_url = local.tekton_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis @@ -67,7 +67,7 @@ module "cloud_build_parser" { source = "../fourkeys-cloud-build-parser" count = contains(var.parsers, "cloud-build") ? 1 : 0 project_id = var.project_id - parser_container_url = local.parser_container_urls["cloud-build"] + parser_container_url = local.cloud_build_parser_url region = var.region fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis From 411487f7043cd1b6a8e5048331204a5d2c9440ff Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 16 Aug 2022 13:37:07 -0700 Subject: [PATCH 03/21] align example with module changes --- terraform/example/main.tf | 5 ----- terraform/example/terraform.tfvars.example | 11 +---------- terraform/example/variables.tf | 6 ------ 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/terraform/example/main.tf b/terraform/example/main.tf index ca8bf1a6..272d1cb5 100644 --- a/terraform/example/main.tf +++ b/terraform/example/main.tf @@ -2,12 +2,7 @@ module "fourkeys" { source = "../modules/fourkeys" project_id = var.project_id enable_apis = var.enable_apis - enable_build_images = var.enable_build_images region = var.region bigquery_region = var.bigquery_region parsers = var.parsers - # Uncomment the following container url variables if enable_build_images is set to false: - # event_handler_container_url = var.event_handler_container_url - # dashboard_container_url = var.dashboard_container_url - # parser_container_urls = var.parser_container_urls } diff --git a/terraform/example/terraform.tfvars.example b/terraform/example/terraform.tfvars.example index 993c61be..7fe2f0ef 100644 --- a/terraform/example/terraform.tfvars.example +++ b/terraform/example/terraform.tfvars.example @@ -1,14 +1,5 @@ project_id = "" enable_apis = -enable_build_images = region = "" bigquery_region = "" -parsers = ["", ""] - -# The variables below are only required if enable_build_images is set to false - -# event_handler_container_url = "" -# dashboard_container_url = "" -# parser_container_urls = { -# '': '' -# } +parsers = ["", ""] \ No newline at end of file diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf index 4e578c53..e26ee5a9 100644 --- a/terraform/example/variables.tf +++ b/terraform/example/variables.tf @@ -9,12 +9,6 @@ variable "enable_apis" { default = false } -variable "enable_build_images" { - type = bool - description = "Toggle to build fourkeys images and upload to container registry. If set to false, URLs for images must be provided via the container_url variables" - default = true -} - variable "region" { type = string default = "us-central1" From d1b47030983c96e02019837cae259f0fc87854e0 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 16 Aug 2022 15:44:48 -0700 Subject: [PATCH 04/21] remove run step from cloudbuild.yaml's --- bq-workers/github-parser/cloudbuild.yaml | 10 ---------- dashboard/cloudbuild.yaml | 10 ---------- event-handler/cloudbuild.yaml | 12 ------------ 3 files changed, 32 deletions(-) diff --git a/bq-workers/github-parser/cloudbuild.yaml b/bq-workers/github-parser/cloudbuild.yaml index d69cb167..d134c545 100644 --- a/bq-workers/github-parser/cloudbuild.yaml +++ b/bq-workers/github-parser/cloudbuild.yaml @@ -25,16 +25,6 @@ steps: waitFor: build id: push -- # Deploy to Cloud Run - name: google/cloud-sdk - args: ['gcloud', 'run', 'deploy', 'github-parser', - '--image', 'gcr.io/$PROJECT_ID/github-parser:${_TAG}', - '--region', '${_REGION}', - '--platform', 'managed' - ] - id: deploy - waitFor: push - images: [ 'gcr.io/$PROJECT_ID/github-parser:${_TAG}' ] diff --git a/dashboard/cloudbuild.yaml b/dashboard/cloudbuild.yaml index 0affe2f3..33938cff 100644 --- a/dashboard/cloudbuild.yaml +++ b/dashboard/cloudbuild.yaml @@ -24,16 +24,6 @@ steps: args: ['push', 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}'] id: push -- # Deploy to Cloud Run - name: google/cloud-sdk - args: ['gcloud', 'run', 'deploy', 'fourkeys-grafana-dashboard', - '--image', 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}', - '--region', '${_REGION}', - '--platform', 'managed', '--port', '3000', - '--allow-unauthenticated' - ] - id: deploy - # Read more about substitutions # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values substitutions: diff --git a/event-handler/cloudbuild.yaml b/event-handler/cloudbuild.yaml index 1a052367..3a20042d 100644 --- a/event-handler/cloudbuild.yaml +++ b/event-handler/cloudbuild.yaml @@ -24,18 +24,6 @@ steps: waitFor: build id: push -- # Deploy to Cloud Run - name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim - args: ['gcloud', 'run', 'deploy', 'event-handler', - '--image', 'gcr.io/$PROJECT_ID/event-handler:${_TAG}', - '--region', '${_REGION}', - '--platform', 'managed', - '--allow-unauthenticated', - '--set-env-vars', 'PROJECT_NAME=$PROJECT_ID' - ] - id: deploy - waitFor: push - images: [ 'gcr.io/$PROJECT_ID/event-handler:${_TAG}' ] From abe26f02c040bd0d6f82d89c3113b8a537538ecc Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 16 Aug 2022 15:45:14 -0700 Subject: [PATCH 05/21] removed unfinished step in module readme --- terraform/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index 53a60e5e..d1249eac 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -95,10 +95,3 @@ To test your Four Keys deployment, you can generate mock data that simulates eve ```sql SELECT * FROM four_keys.events_raw WHERE source = 'githubmock'; ``` -## Updating Cloud Run Services -TODO: replace/rewrite - -When an image is updated in your project's container, run the following to recreate the corresponding Cloud Run Service via gcloud: -``sh -gcloud run services update RUNSERVICENAME --image gcr.io/cloudbuild-fio-b549/:latest -`` From 30e3dfdbabf1f36be2169941a474547cb87deb1c Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 16 Aug 2022 15:46:05 -0700 Subject: [PATCH 06/21] update readmev2 with manual build steps --- setup/READMEv2.md | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/setup/READMEv2.md b/setup/READMEv2.md index d38991a9..2e61b9ac 100644 --- a/setup/READMEv2.md +++ b/setup/READMEv2.md @@ -1,7 +1,8 @@ # Installation guide This guide describes how to set up Four Keys with your GitHub or GitLab project. The main steps are: -1. Forking this repository +1. Forking or cloning this repository +1. Building required images with Cloud Build 1. Providing values for required Terraform variables 1. Executing Terraform to deploy resources 1. Generating sample data (optional) @@ -12,20 +13,48 @@ This guide describes how to set up Four Keys with your GitHub or GitLab project. # Before you begin To deploy Four Keys with Terraform, you will first need: -> TODO: list specific permissions instead of OWNER * A Google Cloud project with billing enabled * The owner role assigned to you on the project * The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. + +You will also need to clone this repository to your local host: + +```sh +git clone https://github.com/GoogleCloudPlatform/fourkeys.git && +cd ./fourkeys/ +``` + ---- -# Deploying with Terraform + +# Build required container images + +Four Keys deploys containerized applications on Cloud Run using corresponding container images for the dashboard, event handler, and each of the services that will connect to Four Keys. By default, Terraform will set up Cloud Run services referencing containers uploaded to your project's container registry with the default names indicated in these steps: + +1. Set an environment variable indicating your Google Cloud project ID: + ```sh + export PROJECT_ID="YOUR_PROJECT_ID" + ``` +1. Build the container for the event handler: + ```sh + gcloud builds submit ./event-handler --tag=gcr.io/${PROJECT_ID}/event-handler + ``` +1. Build the container for the dashboard: + ```sh + gcloud builds submit ./dashboard --tag=gcr.io/${PROJECT_ID}/fourkeys-grafana-dashboard + ``` +1. Build the container(s) for desired service(s). See [/bq-workers](https://github.com/GoogleCloudPlatform/fourkeys/tree/main/bq-workers) for images available. For example, Github: + ```sh + gcloud builds submit ./bq-workers/github-parser --tag=gcr.io/${PROJECT_ID}/github-parser + ``` + +# Deploy with Terraform ## Prepare the code -1. Clone or fork the Four Keys git repository and change your current working directory to `terraform/example` +1. Change your working directory to terraform/example ```sh - git clone https://github.com/GoogleCloudPlatform/fourkeys.git && - cd fourkeys/terraform/example + cd terraform/example ``` The `example` directory has a `main.tf` file that deploys Four Keys' resources via a single Terraform module. The parameters are populated by the variables declared in `variables.tf`. @@ -47,7 +76,7 @@ To deploy Four Keys with Terraform, you will first need: ```sh terraform apply ``` -Once complete, your Four Keys infrastructure is in-place to receive and process events. +Once complete, your Four Keys infrastructure will be in-place to receive and process events. ---- # Generating mock data @@ -89,5 +118,3 @@ To test your Four Keys deployment, you can generate mock data that simulates eve ```sql SELECT * FROM four_keys.events_raw WHERE source = 'githubmock'; ``` - ----- \ No newline at end of file From 5284e846f3e5bbcfa4a7b9e90b4f1fef4f4803ac Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 25 Aug 2022 10:57:35 -0700 Subject: [PATCH 07/21] update e2e terraform test with updated var --- ci/e2e_terraform_module_test.cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/e2e_terraform_module_test.cloudbuild.yaml b/ci/e2e_terraform_module_test.cloudbuild.yaml index 0d6c30f9..68dfe323 100644 --- a/ci/e2e_terraform_module_test.cloudbuild.yaml +++ b/ci/e2e_terraform_module_test.cloudbuild.yaml @@ -53,7 +53,7 @@ steps: bigquery_region="US" event_handler_container_url="gcr.io/$_TARGET_PROJECT/event-handler:$SHORT_SHA" dashboard_container_url="gcr.io/$_TARGET_PROJECT/dashboard:$SHORT_SHA" - parser_container_urls={"github":"gcr.io/$_TARGET_PROJECT/github-parser:$SHORT_SHA"} + github_parser_url="gcr.io/$_TARGET_PROJECT/github-parser:$SHORT_SHA" } EOF waitFor: ['-'] From 53658d4c5f820a8c7e0e25e408d5c83efa453ad1 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 6 Oct 2022 12:17:04 -0700 Subject: [PATCH 08/21] remove dupe module --- terraform/modules/fourkeys/parsers.tf | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/terraform/modules/fourkeys/parsers.tf b/terraform/modules/fourkeys/parsers.tf index 727b69c1..65b31c57 100644 --- a/terraform/modules/fourkeys/parsers.tf +++ b/terraform/modules/fourkeys/parsers.tf @@ -50,19 +50,6 @@ module "pagerduty_parser" { ] } -module "pagerduty_parser" { - source = "../fourkeys-pagerduty-parser" - count = contains(var.parsers, "pagerduty") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.parser_container_urls["pagerduty"] - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - module.fourkeys_images - ] -} - module "tekton_parser" { source = "../fourkeys-tekton-parser" count = contains(var.parsers, "tekton") ? 1 : 0 From c263daeadf39b78e0fb38db33e622495de9e209a Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 6 Oct 2022 12:59:59 -0700 Subject: [PATCH 09/21] add build steps and adjust yaml subs --- dashboard/cloudbuild.yaml | 1 - event-handler/cloudbuild.yaml | 2 ++ terraform/README.md | 23 +++++++++++++++-------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/dashboard/cloudbuild.yaml b/dashboard/cloudbuild.yaml index 33938cff..875e449c 100644 --- a/dashboard/cloudbuild.yaml +++ b/dashboard/cloudbuild.yaml @@ -28,4 +28,3 @@ steps: # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values substitutions: _TAG: latest - _REGION: us-central1 diff --git a/event-handler/cloudbuild.yaml b/event-handler/cloudbuild.yaml index 3a20042d..36c752ed 100644 --- a/event-handler/cloudbuild.yaml +++ b/event-handler/cloudbuild.yaml @@ -27,3 +27,5 @@ steps: images: [ 'gcr.io/$PROJECT_ID/event-handler:${_TAG}' ] +substitutions: + _TAG: latest diff --git a/terraform/README.md b/terraform/README.md index d1249eac..5dbfc0dd 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -37,15 +37,22 @@ To deploy Four Keys with Terraform, you will first need: * The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. ## Deploying with Terraform -#TODO: Replace step 1 with rewrite -1. Terraform will presume that the project you're using will have the relavant images in the container registry. Build the following: - - dashboard - - event handler - - parsers you plan on using -1. Clone the fourkeys git repository, or copy the files in the `terraform/example` directory to your working directory - -1. Rename `terraform.tfvars.example` to `terraform.tfvars` +1. Clone the fourkeys git repository and change into the root directory + ``` + git clone https://github.com/GoogleCloudPlatform/fourkeys.git && cd fourkeys + ``` + +1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: + ``` + gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml && \ + gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml + ``` + +1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` + ``` + cd terraform/example && mv terraform.tfvars.example terraform.tfvars + ``` 1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. To accept the default value of a variable indicated in `variables.tf`, exclude it from `terraform.tfvars` From e43bcd87bb71221a1a50a03780f9ada8f3cd9712 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 6 Oct 2022 13:00:28 -0700 Subject: [PATCH 10/21] add parser build step and add shared yaml --- bq-workers/parsers.cloudbuild.yaml | 33 ++++++++++++++++++++++++++++++ terraform/README.md | 5 +++++ 2 files changed, 38 insertions(+) create mode 100644 bq-workers/parsers.cloudbuild.yaml diff --git a/bq-workers/parsers.cloudbuild.yaml b/bq-workers/parsers.cloudbuild.yaml new file mode 100644 index 00000000..b3471f3f --- /dev/null +++ b/bq-workers/parsers.cloudbuild.yaml @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- # Build parser image + name: gcr.io/cloud-builders/docker:latest + dir: ${_SERVICE}-parser + args: ['build', + '--tag=gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}', '.'] + id: build + +- # Push the container image to Container Registry + name: gcr.io/cloud-builders/docker + args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}'] + waitFor: build + id: push + +images: [ + 'gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}' +] +substitutions: + _TAG: latest diff --git a/terraform/README.md b/terraform/README.md index 5dbfc0dd..3812809e 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -49,6 +49,11 @@ To deploy Four Keys with Terraform, you will first need: gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml ``` +1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. Github for example: + ``` + gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --substitutions=_SERVICE=github + ``` + 1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` ``` cd terraform/example && mv terraform.tfvars.example terraform.tfvars From b744c6b08a058a4194d8cc5b146086a92e91e3a0 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 6 Oct 2022 13:03:29 -0700 Subject: [PATCH 11/21] rewording --- terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index 3812809e..f7028c05 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -59,9 +59,9 @@ To deploy Four Keys with Terraform, you will first need: cd terraform/example && mv terraform.tfvars.example terraform.tfvars ``` -1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. To accept the default value of a variable indicated in `variables.tf`, exclude it from `terraform.tfvars` +1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. Values not defined in `terraform.tfvars` will use default values defined in `variables.tf` -1. Run the following commands from the `example` directory, or your working directory: +1. Run the following commands from the `example` directory: `terraform init` to inialize Terraform and download the module From d524488de3a1dcd1ec1d8a22fe45fac496fd0328 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 12:36:37 -0700 Subject: [PATCH 12/21] have example enable apis --- terraform/example/terraform.tfvars.example | 1 - terraform/example/variables.tf | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/example/terraform.tfvars.example b/terraform/example/terraform.tfvars.example index 7fe2f0ef..399b34e9 100644 --- a/terraform/example/terraform.tfvars.example +++ b/terraform/example/terraform.tfvars.example @@ -1,5 +1,4 @@ project_id = "" -enable_apis = region = "" bigquery_region = "" parsers = ["", ""] \ No newline at end of file diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf index e26ee5a9..cf5b6ce0 100644 --- a/terraform/example/variables.tf +++ b/terraform/example/variables.tf @@ -6,8 +6,7 @@ variable "project_id" { variable "enable_apis" { type = bool description = "Toggle to include required APIs." - default = false -} + default = true variable "region" { type = string From 3324e438c8dd38678f78599ad5683c256c7b77ce Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 12:37:56 -0700 Subject: [PATCH 13/21] missing bracket --- terraform/example/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf index cf5b6ce0..aece5d21 100644 --- a/terraform/example/variables.tf +++ b/terraform/example/variables.tf @@ -7,6 +7,7 @@ variable "enable_apis" { type = bool description = "Toggle to include required APIs." default = true +} variable "region" { type = string From 0989b154b4aea15d6900b8b1b6dd326d21b266f8 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 13:35:01 -0700 Subject: [PATCH 14/21] grammar --- terraform/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index f7028c05..cef5f90b 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -16,7 +16,7 @@ module "fourkeys" { } ``` -The example above will deploy Four Keys with a Github parser for Github events. See the `terraform/example` directory for full example and options. +The example above will deploy Four Keys with a GitHub parser for GitHub events. See the `terraform/example` directory for full example and options. Alternatively, you can fork the fourkeys project and deploy as a local module from the `terraform/example` directory: @@ -49,7 +49,7 @@ To deploy Four Keys with Terraform, you will first need: gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml ``` -1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. Github for example: +1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: ``` gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --substitutions=_SERVICE=github ``` @@ -71,9 +71,9 @@ To deploy Four Keys with Terraform, you will first need: ## Generating mock data -To test your Four Keys deployment, you can generate mock data that simulates events from a Github repository. +To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. -1. Export your event handler URL an environment variable. This the webhook URL that will receive events: +1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: ```sh export WEBHOOK=`gcloud run services list | grep event-handler | awk '{print $4}'` From 187afc5dbe76745c9d906faa31f972206dcdbe26 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 13:35:17 -0700 Subject: [PATCH 15/21] add in a wait for services --- terraform/modules/fourkeys/dashboard.tf | 3 ++- terraform/modules/fourkeys/event-handler.tf | 6 ++++++ terraform/modules/fourkeys/parsers.tf | 18 ++++++++++++------ terraform/modules/fourkeys/services.tf | 6 ++++++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/terraform/modules/fourkeys/dashboard.tf b/terraform/modules/fourkeys/dashboard.tf index 57e17fcc..d90cfe60 100644 --- a/terraform/modules/fourkeys/dashboard.tf +++ b/terraform/modules/fourkeys/dashboard.tf @@ -29,7 +29,8 @@ resource "google_cloud_run_service" "dashboard" { } autogenerate_revision_name = true depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } diff --git a/terraform/modules/fourkeys/event-handler.tf b/terraform/modules/fourkeys/event-handler.tf index 3d2ef8ad..ae81cb9c 100644 --- a/terraform/modules/fourkeys/event-handler.tf +++ b/terraform/modules/fourkeys/event-handler.tf @@ -22,6 +22,9 @@ resource "google_cloud_run_service" "event_handler" { } autogenerate_revision_name = true + depends_on = [ + time_sleep.wait_for_services + ] } resource "google_cloud_run_service_iam_binding" "event_handler_noauth" { @@ -39,6 +42,9 @@ resource "google_secret_manager_secret" "event_handler" { replication { automatic = true } + depends_on = [ + time_sleep.wait_for_services + ] } resource "random_id" "event_handler_random_value" { diff --git a/terraform/modules/fourkeys/parsers.tf b/terraform/modules/fourkeys/parsers.tf index 65b31c57..48835e70 100644 --- a/terraform/modules/fourkeys/parsers.tf +++ b/terraform/modules/fourkeys/parsers.tf @@ -7,7 +7,8 @@ module "circleci_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } @@ -20,7 +21,8 @@ module "github_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } @@ -33,7 +35,8 @@ module "gitlab_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } @@ -46,7 +49,8 @@ module "pagerduty_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } @@ -59,7 +63,8 @@ module "tekton_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } @@ -72,6 +77,7 @@ module "cloud_build_parser" { fourkeys_service_account_email = google_service_account.fourkeys.email enable_apis = var.enable_apis depends_on = [ - module.fourkeys_images + module.fourkeys_images, + time_sleep.wait_for_services ] } \ No newline at end of file diff --git a/terraform/modules/fourkeys/services.tf b/terraform/modules/fourkeys/services.tf index 038d06b5..26d4c47e 100644 --- a/terraform/modules/fourkeys/services.tf +++ b/terraform/modules/fourkeys/services.tf @@ -4,4 +4,10 @@ resource "google_project_service" "fourkeys_services" { for_each = toset(local.services) service = each.value disable_on_destroy = false +} + +resource "time_sleep" "wait_for_services" { + depends_on = [google_project_service.fourkeys_services] + + create_duration = "30s" } \ No newline at end of file From 119c20df3b682ea0e6b101aa184f9e454c58a8a9 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 13:52:38 -0700 Subject: [PATCH 16/21] update readmev2 to match terraform readme --- setup/READMEv2.md | 66 +++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/setup/READMEv2.md b/setup/READMEv2.md index 8b9a2249..40a12b9c 100644 --- a/setup/READMEv2.md +++ b/setup/READMEv2.md @@ -14,15 +14,15 @@ This guide describes how to set up Four Keys with your GitHub or GitLab project. # Before you begin To deploy Four Keys with Terraform, you will first need: + * A Google Cloud project with billing enabled * The owner role assigned to you on the project * The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. -You will also need to clone this repository to your local host: +You will also need to clone this repository to your local machine: ```sh -git clone https://github.com/GoogleCloudPlatform/fourkeys.git && -cd ./fourkeys/ +git clone https://github.com/GoogleCloudPlatform/fourkeys.git && cd fourkeys ``` ---- @@ -37,73 +37,59 @@ Four Keys deploys containerized applications on Cloud Run using corresponding co ``` 1. Build the container for the event handler: ```sh - gcloud builds submit ./event-handler --tag=gcr.io/${PROJECT_ID}/event-handler + gcloud builds submit ./event-handler --tag=gcr.io/${PROJECT_ID}/event-handler --project $PROJECT_ID ``` 1. Build the container for the dashboard: ```sh - gcloud builds submit ./dashboard --tag=gcr.io/${PROJECT_ID}/fourkeys-grafana-dashboard - ``` -1. Build the container(s) for desired service(s). See [/bq-workers](https://github.com/GoogleCloudPlatform/fourkeys/tree/main/bq-workers) for images available. For example, Github: - ```sh - gcloud builds submit ./bq-workers/github-parser --tag=gcr.io/${PROJECT_ID}/github-parser + gcloud builds submit ./dashboard --tag=gcr.io/${PROJECT_ID}/fourkeys-grafana-dashboard --project $PROJECT_ID ``` +1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: + ```sh + gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github + ``` # Deploy with Terraform -## Prepare the code +1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` + ``` + cd terraform/example && mv terraform.tfvars.example terraform.tfvars + ``` -1. Change your working directory to terraform/example +1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. Values not defined in `terraform.tfvars` will use default values defined in `variables.tf` - ```sh - cd terraform/example - ``` - The `example` directory has a `main.tf` file that deploys Four Keys' resources via a single Terraform module. The parameters are populated by the variables declared in `variables.tf`. +1. Run the following commands from the `example` directory: -2. Rename `terraform.tfvars.example` to `terraform.tfvars` -3. Edit in values for the required variables. To accept the default value of a variable indicated in `variables.tf`, exclude it from `terraform.tfvars` + `terraform init` to inialize Terraform and download the module -## Initialize and apply the Terraform + `terraform plan` to preview changes. + + `terraform apply` to deploy the resources. -1. Initialize the Terraform: - ```sh - terraform init - ``` -1. Before applying the Terraform, preview changes and catch any errors in your configuration: - - ```sh - terraform plan - ``` -1. Deploy the resources to your Google Cloud Project: - ```sh - terraform apply - ``` Once complete, your Four Keys infrastructure will be in-place to receive and process events. ----- # Generating mock data -To test your Four Keys deployment, you can generate mock data that simulates events from a Github repository. +To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. -1. Export your event handler URL to an environment variable. This is the webhook URL that will receive events: +1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: ```sh - export WEBHOOK=`gcloud run services list --project= --format 'value(status.url)' --filter=metadata.name:event-handler` + export WEBHOOK=`gcloud run services list | grep event-handler | awk '{print $4}'` ``` 1. Export your event handler secret to an environment variable. This is the secret used to authenticate events sent to the webhook: ```sh - export SECRET=`gcloud secrets versions access --project= --secret=event-handler 1` - - ``` + export SECRET=`gcloud secrets versions access 1 --secret=event-handler` + ``` 1. From the root of the fourkeys project run: ```sh - python3 data_generator/generate-data.py --vc_system=github + python3 data-generator/generate_data.py --vc_system=github ``` - The data generated will run through the pipeline that the Terraform provisioned: + You can see these events being run through the pipeline: * The event handler logs show successful requests * The Pub/Sub topic show messages posted * The BigQuery GitHub parser show successful requests From a929d4543f7c9fdca7bb5ece6555a1a6bfec8a05 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Thu, 3 Nov 2022 13:59:14 -0700 Subject: [PATCH 17/21] match terraform and setup READMEs --- setup/READMEv2.md | 53 ++++++++++++++++++--------------------------- terraform/README.md | 17 ++++++++++----- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/setup/READMEv2.md b/setup/READMEv2.md index 40a12b9c..21578f5a 100644 --- a/setup/READMEv2.md +++ b/setup/READMEv2.md @@ -7,11 +7,10 @@ This guide describes how to set up Four Keys with your GitHub or GitLab project. 1. Providing values for required Terraform variables 1. Executing Terraform to deploy resources 1. Generating sample data (optional) -1. Integrating your repository to send data to your Four Keys deployment. -> Alternatively, to deploy Four Keys as a remote Terraform module, see `terraform/modules/fourkeys/README.md` ----- -# Before you begin +> Alternatively, to deploy Four Keys as a remote Terraform module, see [`terraform/modules/fourkeys/README.md`](../terraform/modules/fourkeys/README.md) + +## Before you begin To deploy Four Keys with Terraform, you will first need: @@ -19,37 +18,29 @@ To deploy Four Keys with Terraform, you will first need: * The owner role assigned to you on the project * The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. -You will also need to clone this repository to your local machine: - -```sh -git clone https://github.com/GoogleCloudPlatform/fourkeys.git && cd fourkeys -``` - ----- - -# Build required container images - -Four Keys deploys containerized applications on Cloud Run using corresponding container images for the dashboard, event handler, and each of the services that will connect to Four Keys. By default, Terraform will set up Cloud Run services referencing containers uploaded to your project's container registry with the default names indicated in these steps: +## Deploying with Terraform 1. Set an environment variable indicating your Google Cloud project ID: ```sh export PROJECT_ID="YOUR_PROJECT_ID" ``` -1. Build the container for the event handler: - ```sh - gcloud builds submit ./event-handler --tag=gcr.io/${PROJECT_ID}/event-handler --project $PROJECT_ID - ``` -1. Build the container for the dashboard: - ```sh - gcloud builds submit ./dashboard --tag=gcr.io/${PROJECT_ID}/fourkeys-grafana-dashboard --project $PROJECT_ID - ``` + +1. Clone the fourkeys git repository and change into the root directory + ``` + git clone https://github.com/GoogleCloudPlatform/fourkeys.git && cd fourkeys + ``` + +1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: + ``` + gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID && \ + gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID + ``` + 1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: - ```sh + ``` gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github ``` -# Deploy with Terraform - 1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` ``` cd terraform/example && mv terraform.tfvars.example terraform.tfvars @@ -65,22 +56,20 @@ Four Keys deploys containerized applications on Cloud Run using corresponding co `terraform apply` to deploy the resources. -Once complete, your Four Keys infrastructure will be in-place to receive and process events. - -# Generating mock data +## Generating mock data To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. 1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: ```sh - export WEBHOOK=`gcloud run services list | grep event-handler | awk '{print $4}'` + export WEBHOOK=`gcloud run services list --project $PROJECT_ID | grep event-handler | awk '{print $4}'` ``` 1. Export your event handler secret to an environment variable. This is the secret used to authenticate events sent to the webhook: ```sh - export SECRET=`gcloud secrets versions access 1 --secret=event-handler` + export SECRET=`gcloud secrets versions access 1 --secret=event-handler --project $PROJECT_ID` ``` 1. From the root of the fourkeys project run: @@ -97,7 +86,7 @@ To test your Four Keys deployment, you can generate mock data that simulates eve 1. View the generated data in the `events_raw` table in with bq: ```sh - bq query 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' + bq query --project_id $PROJECT_ID 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' ``` Or query the table directly in [BigQuery](https://console.cloud.google.com/bigquery): diff --git a/terraform/README.md b/terraform/README.md index cef5f90b..e4c3c06c 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -38,6 +38,11 @@ To deploy Four Keys with Terraform, you will first need: ## Deploying with Terraform +1. Set an environment variable indicating your Google Cloud project ID: + ```sh + export PROJECT_ID="YOUR_PROJECT_ID" + ``` + 1. Clone the fourkeys git repository and change into the root directory ``` git clone https://github.com/GoogleCloudPlatform/fourkeys.git && cd fourkeys @@ -45,13 +50,13 @@ To deploy Four Keys with Terraform, you will first need: 1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: ``` - gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml && \ - gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml + gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID && \ + gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID ``` 1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: ``` - gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --substitutions=_SERVICE=github + gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github ``` 1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` @@ -76,13 +81,13 @@ To test your Four Keys deployment, you can generate mock data that simulates eve 1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: ```sh - export WEBHOOK=`gcloud run services list | grep event-handler | awk '{print $4}'` + export WEBHOOK=`gcloud run services list --project $PROJECT_ID | grep event-handler | awk '{print $4}'` ``` 1. Export your event handler secret to an environment variable. This is the secret used to authenticate events sent to the webhook: ```sh - export SECRET=`gcloud secrets versions access 1 --secret=event-handler` + export SECRET=`gcloud secrets versions access 1 --secret=event-handler --project $PROJECT_ID` ``` 1. From the root of the fourkeys project run: @@ -99,7 +104,7 @@ To test your Four Keys deployment, you can generate mock data that simulates eve 1. View the generated data in the `events_raw` table in with bq: ```sh - bq query 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' + bq query --project_id $PROJECT_ID 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' ``` Or query the table directly in [BigQuery](https://console.cloud.google.com/bigquery): From 420eb571b7ceaafbe9a4319fca02eb2a2406e7a2 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 15 Nov 2022 14:15:29 -0800 Subject: [PATCH 18/21] remove old readme --- setup/README.md | 236 ------------------------------------------------ 1 file changed, 236 deletions(-) delete mode 100644 setup/README.md diff --git a/setup/README.md b/setup/README.md deleted file mode 100644 index 095ad964..00000000 --- a/setup/README.md +++ /dev/null @@ -1,236 +0,0 @@ -# Installation guide - -This guide describes how to set up Four Keys with your GitHub or GitLab project. The main steps are: - -1. [Running the setup script](#running-the-setup-script) -1. Integrating with your GitHub or GitLab repo by: - 1. [Collecting changes data](#collecting-changes-data) - 1. [Collecting deployment data](#collecting-deployment-data) - 1. [Collecting incident data](#collecting-incident-data) - -## Before you begin -> We recommend using [Cloud Shell](https://cloud.google.com/shell) to install Four Keys -1. Install [GCloud SDK](https://cloud.google.com/sdk/install). -1. Install [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli). -1. You must be owner on a Google Cloud project that has billing enabled. You can either use a currently active project or create a new project specifically to use with Four Keys. - -> :information_source: To create a new project using the same billing account as your currently-active gcloud project, run the following commands: -> ```sh -> export PARENT_PROJECT=$(gcloud config get-value project) -> export PARENT_FOLDER=$(gcloud projects describe ${PARENT_PROJECT} --format="value(parent.id)") -> export BILLING_ACCOUNT=$(gcloud beta billing projects describe ${PARENT_PROJECT} --format="value(billingAccountName)") -> export FOURKEYS_PROJECT=$(printf "fourkeys-%06d" $((RANDOM%999999))) -> gcloud projects create ${FOURKEYS_PROJECT} --folder=${PARENT_FOLDER} -> gcloud beta billing projects link ${FOURKEYS_PROJECT} --billing-account=${BILLING_ACCOUNT} -> echo "project created: "$FOURKEYS_PROJECT -> -> ``` - - -## Running the setup script - -1. Run the following setup script from the top-level directory of this repository: - - ```bash - cd setup - script setup.log -c ./setup.sh - ``` -1. Answer the setup script's questions: - - * Enter the project ID and region information for the project in which you wish to install Four Keys - * Choose the event sources to configure... - * Which version control system are you using? - * Choose the appropriate option for your VCS, or choose "other" to skip VCS integration - * Which CI/CD system are you using? - * Choose the appropriate option for your CICD system, or choose "other" to skip CICD integration - * _(see `/README.md#extending-to-other-event-sources` to integrate event sources not available during setup)_ - * Would you like to generate mock data? (y/N) - * If you select yes, a script will run through and send mock GitLab or GitHub events to your event-handler. This will populate your dashboard with mock data. The mock data will include the work "mock" in the source. You can generate mock data without using the setup script. See [Generating mock data](../README.md). - * To exclude the mock data from the dashboard, update the SQL script to filter out any source with the word mock in it by adding: `WHERE source not like "%mock"`. - -### Making changes -At some point after running the setup script, you may want to make modifications to your infrastructure. Or, the Four Keys repo itself may be updated with a new configuration. If you make changes to your resources outside of Terraform, they will not be tracked and cannot be managed by Terraform. This includes pub/sub topics, subscriptions, permissions, service accounts, services, etc. Therefore, it's recommended to make all infrastructure changes by updating your Terraform files and re-running Terraform, using `terraform apply`. You'll be prompted to confirm the planned changes; review them carefully, then type `yes` to proceed. -> Tip: The configurations in this repo will continue to evolve over time; if you want to be able to apply ongoing updates, **don't modify the tracked Terraform files**. Instead, consider using [Terraform Override Files](https://www.terraform.io/docs/language/files/override.html), which will allow you to customize the infrastructure to your needs without introducing potential merge conflicts the next time you pull from upstream. - -### The setup explained -The setup script does many things to help create the service architecture described in the `README.md`. These include a little bash scripting and a lot of [Terraform](https://www.terraform.io/intro/). - -Step by step, here's what's happening: -1. `setup.sh` starts by collecting information from the system and the user to determine a number of configuration variables that will be provided to Terraform. -1. It sets several environment variables, and writes a `terraform.tfvars` file to disk, containing inputs to Terraform. -1. Then it invokes `install.sh`, which is responsible for provisioning the infrastructure. -1. `install.sh` runs `gcloud builds submit` commands to build the application containers that will be used in Cloud Run services. -1. Then it invokes Terraform, which processes the configuration files (ending in `.tf`) to provision all of the necessary infrastructure into the speficied Cloud project. -1. If you've chosen to generate mock data, the script then calls the ["data generator" python application](/data-generator/) to submit several synthetic webhook events to the event-handler service that was just created. -1. Finally, the script prints information about next steps, including configuring webhooks and visiting the dashboard. - -### Managing Terraform State -Terraform maintains information about infrastucture in persistent state storage, known as a backend. By default, this is maintained in a file named `terraform.tfstate`, saved to the same directory that Terraform is executed from. This local backend is fine for a one-time setup, but if you plan to maintain and use your Four Keys infrastructure, it's recommended to choose a remote backend. (Alternatively, you may choose to use Terraform only for the initial setup, and then use other tools--like `gcloud` or the Cloud Console--for ongoing modifications.) - -> To learn how to use a remote backend for robust storage of Terraform state, see: [Terraform Language: Backends](https://www.terraform.io/docs/language/settings/backends/index.html) - -### Purging resources created by Terraform -If something goes wrong during Terraform setup, you may be able to run `terraform destroy` to delete the resources that were created. However, it's possible for the Terraform state to become inconsistent with your project, leaving Terraform unaware of resources (yet their existance will prevent subsequent installations from working). If that happens, the best option is usually to delete the GCP project and start a new one. If that's not possible, you can force-remove all of the four keys resources in your project by running: -```shell -./ci/project_cleaner.sh --project= -``` - -## Integrating with a live repo -The setup script can create mock data, but it cannot integrate automatically with live projects. To measure your team's performance, you need to integrate to your live GitHub or GitLab repo that has ongoing deployments. You can then measure the four key metrics, and experiment with how changes, successful deployments, and failed deployments affect your metrics. - -To integrate Four Keys with a live repo, you need to: - -1. [Collect changes data](#collecting-changes-data) -1. [Collect deployment data](#collecting-deployment-data) -1. [Collect incident data](#collecting-incident-data) - -## Migrating from an earlier version of The Four Keys -If you have an existing installation of The Four Keys, created using the now-deprecated [bash-based setup process](deprecated/), and you want to be able to keep your installation up-to-date with new upstream releases, you'll need to put your cloud resources under Terraform control. The easiest way to do this is to destroy all existing resources and let Terraform create new ones that it will then manage going forward. Here's the process to do that (adapt as needed for your specific installation): -1. [Export the data](https://cloud.google.com/bigquery/docs/exporting-data#console) from `events_raw` - * _If you exported your data to a bucket in a project that you plan to delete, be sure to download it before deleting the project!_ -1. Delete existing cloud resources - * If you have a project dedicated to Four Keys, you can simply delete that project -1. Run `setup.sh` in this folder - * When configuring the installation, choose to not generate mock data -1. When the setup is complete: - 1. Use the newly-generated webhook URL and secret to reconfigure webhook deliveries from your VCS/CICD systems - 1. Import the `events_raw` data: - 1. [Load the data into a temporary table](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-csv#console) named `events_raw_import` - * You may need to manually specify the schema (and delete the column headers) when importing - 1. Copy the imported data into the `events_raw` table - * `INSERT INTO events_raw (SELECT * FROM events_raw_import)` - 1. Delete the temporary table - -### Collecting changes data - -#### GitHub instructions - -1. Start with your GitHub repo -1. Navigate to your repo (or forked repo) and click **Settings**. -1. Select **Webhooks** from the left hand side. -1. Click **Add Webhook**. -1. Get the Event Handler endpoint for your Four Keys service: - ```bash - echo $(terraform output -raw event_handler_endpoint) - ``` -1. In the **Add Webhook** interface use the Event Handler endpoint for **Payload URL**. -1. Run the following command to get the secret from Google Secrets Manager: - ```bash - echo $(terraform output -raw event_handler_secret) - ``` -1. Put the secret in the box labelled **Secret**. -1. For **Content Type**, select **application/json**. -1. Select **Send me everything**. -1. Click **Add Webhook**. - -#### GitLab instructions - -1. Navigate to your repo and click **Settings**. -1. Select **Webhooks** from the menu. -1. Get the Event Handler endpoint for your Four Keys service by running the following: - ```bash - echo $(terraform output -raw event_handler_endpoint) - ``` -1. For **Payload URL**, use the Event Handler endpoint. -1. Run the following command to get the secret from Google Secrets Manager: - ```bash - echo $(terraform output -raw event_handler_secret) - ``` -1. Put the secret in the box labelled **Secret Token**. -1. Select all the checkboxes. -1. Leave the **Enable SSL verification** selected. -1. Click **Add Webhook**. - -### Collecting deployment data - -1. For whichever CI/CD system you are using, set it up to send Webhook events to the event-handler. - -#### Configuring CircleCI to deploy on GitHub or Gitlab merges - -1. Add a `.circleci.yaml` file to your repo. - ``` - version: 2.1 - executors: - default: - ... - jobs: - build: - executor: default - steps: - - run: make build - deploy: - executor: default - steps: - - run: make deploy - workflows: - version: 2 - build_and_deploy_on_master: # A workflow whose name contains 'deploy' will be used in the query to build the deployments view - jobs: - - build: - name: build - filters: &master_filter - branches: - only: master - - deploy: - name: deploy - filters: *master_filter - requires: - - build - ``` - -This setup will trigger a deployment on any `push` to the `master` branch. - -### Collecting incident data - -Four Keys uses GitLab and/or GitHub issues to track incidents. - -#### Creating an incident - -1. Open an issue. -1. Add the tag `Incident`. -1. In the body of the issue, input `root cause: {SHA of the commit}`. - -When the incident is resolved, close the issue. Four Keys will measure the incident from the time of the deployment to when the issue is closed. - -#### Pager Duty Support -If Pager Duty support is enabled (passed via the `parsers` variable), this secret is required and used for verifying Pager Duty events received belong to us. - -To create this secret: - -1. You will need a [Pager Duty General Access REST API Key](https://support.pagerduty.com/docs/api-access-keys#section-generate-a-general-access-rest-api-key). These can only be created by users that are >=Global Admin. -2. Using said API key, [create a webhook subscription](https://developer.pagerduty.com/api-reference/b3A6MjkyNDc4NA-create-a-webhook-subscription). The example below creates an account-wide subscription, but depending on your Four Keys architecture, you could choose to create individual subscriptions per-project or service. - -``` -API_TOKEN= -FOURKEYS_ENDPOINT= -curl-- location-- request POST - 'https://api.pagerduty.com/webhook_subscriptions'-- header - 'Authorization: Token token=${API_TOKEN}'-- header - 'Content-Type: application/json'-- header - 'Accept: application/vnd.pagerduty+json;version=2'-- data - raw '{ - "webhook_subscription": { - "delivery_method": { - "type": "http_delivery_method", - "url": "${FOURKEYS_ENDPOINT}" - }, - "description": "Sends PagerDuty v3 webhook events to DORA metrics.", - "events": [ - "incident.resolved", - "incident.triggered" - ], - "filter": { - "type": "account_reference" - }, - "type": "webhook_subscription" - } - }' -``` - -3. The Pager Duty webhook subscription creation API response will include a secret (_only_ returned on creation). This secret needs to be stored in Secret Manager in your Four Keys project as `pager_duty_secret`. - -``` -SECRET= -echo $SECRET | tr -d '\n' | gcloud beta secrets create pager_duty_secret \ - --replication-policy=automatic \ - --data-file=- -``` From 36825572a80dc78635f00db6db130d7646fba18d Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 15 Nov 2022 14:15:59 -0800 Subject: [PATCH 19/21] rename readmev2 --- setup/{READMEv2.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename setup/{READMEv2.md => README.md} (100%) diff --git a/setup/READMEv2.md b/setup/README.md similarity index 100% rename from setup/READMEv2.md rename to setup/README.md From 1cd3dcd30293aa838e5c2ad8bd23c017482e5b16 Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 15 Nov 2022 14:19:13 -0800 Subject: [PATCH 20/21] remove old setup files --- setup/dashboard/main.tf | 43 ---------- setup/dashboard/outputs.tf | 3 - setup/dashboard/variables.tf | 19 ----- setup/install.sh | 106 ----------------------- setup/main.tf | 19 ----- setup/new_source.sh | 1 - setup/outputs.tf | 12 --- setup/providers.tf | 4 - setup/resource_bigquery.tf | 147 -------------------------------- setup/resource_dashboard.tf | 12 --- setup/resource_event_handler.tf | 71 --------------- setup/resource_parsers.tf | 13 --- setup/variables.tf | 20 ----- 13 files changed, 470 deletions(-) delete mode 100644 setup/dashboard/main.tf delete mode 100644 setup/dashboard/outputs.tf delete mode 100644 setup/dashboard/variables.tf delete mode 100755 setup/install.sh delete mode 100644 setup/main.tf delete mode 100644 setup/new_source.sh delete mode 100644 setup/outputs.tf delete mode 100644 setup/providers.tf delete mode 100644 setup/resource_bigquery.tf delete mode 100644 setup/resource_dashboard.tf delete mode 100644 setup/resource_event_handler.tf delete mode 100644 setup/resource_parsers.tf delete mode 100644 setup/variables.tf diff --git a/setup/dashboard/main.tf b/setup/dashboard/main.tf deleted file mode 100644 index 4762d980..00000000 --- a/setup/dashboard/main.tf +++ /dev/null @@ -1,43 +0,0 @@ -resource "google_cloud_run_service" "dashboard" { - name = "fourkeys-grafana-dashboard" - location = var.google_region - - template { - spec { - containers { - ports { - container_port = 3000 - } - image = "gcr.io/${var.google_project_id}/fourkeys-grafana-dashboard" - env { - name = "PROJECT_NAME" - value = var.google_project_id - } - env { - name = "BQ_REGION" - value = var.bigquery_region - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - metadata { - labels = { "created_by" : "fourkeys" } - } - autogenerate_revision_name = true -} - -resource "google_cloud_run_service_iam_binding" "noauth" { - location = var.google_region - project = var.google_project_id - service = "fourkeys-grafana-dashboard" - - role = "roles/run.invoker" - members = ["allUsers"] - depends_on = [google_cloud_run_service.dashboard] -} diff --git a/setup/dashboard/outputs.tf b/setup/dashboard/outputs.tf deleted file mode 100644 index 521f3721..00000000 --- a/setup/dashboard/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "dashboard_endpoint" { - value = google_cloud_run_service.dashboard.status[0]["url"] -} \ No newline at end of file diff --git a/setup/dashboard/variables.tf b/setup/dashboard/variables.tf deleted file mode 100644 index 09971d8b..00000000 --- a/setup/dashboard/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "google_project_id" { - type = string -} - -variable "google_region" { - type = string -} - -variable "fourkeys_service_account_email" { - type = string -} - -variable "bigquery_region" { - type = string - validation { - condition = can(regex("^(US|EU)$", var.bigquery_region)) - error_message = "The value for 'bigquery_region' must be one of: 'US','EU'." - } -} \ No newline at end of file diff --git a/setup/install.sh b/setup/install.sh deleted file mode 100755 index e9c11ff9..00000000 --- a/setup/install.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script installs Four Keys; it requires several environment -# variables and terraform variables to be set; to set them interactively -# and then launch installation, run `setup.sh`. - - # REQUIRED ENVIRONMENT VARIABLES - # GIT_SYSTEM (e.g. "github") - # CICD_SYSTEM (e.g. "cloud-build") - # PARENT_PROJECT (the project that will orchestrate the install) - # FOURKEYS_PROJECT (the project to install Four Keys to) - # FOURKEYS_REGION (GCP region for cloud resources) - # BIGQUERY_REGION (location for BigQuery resources) - # GENERATE_DATA ["yes"|"no"] - - # REQUIRED TERRAFORM VARIABLES - # google_project_id (FOURKEYS_PROJECT) - # google_region (FOURKEYS_REGION) - # bigquery_region (BIGQUERY_REGION) - # parsers [(list of VCS and CICD parsers to install)] - -set -eEuo pipefail - -# color formatting shortcuts -export GREEN="\033[0;32m" -export NOCOLOR="\033[0m" - -# build service containers (using parent project) and store them in the fourkeys project -echo "••••••••🔑••🔑••🔑••🔑••••••••" -echo "Building containers…" -gcloud services enable cloudbuild.googleapis.com -gcloud services enable containerregistry.googleapis.com --project=${FOURKEYS_PROJECT} -gcloud services enable secretmanager.googleapis.com - -PARENT_PROJECTNUM=$(gcloud projects describe $(gcloud config get-value project) --format='value(projectNumber)') -FOURKEYS_PROJECTNUM=$(gcloud projects describe ${FOURKEYS_PROJECT} --format='value(projectNumber)') -gcloud projects add-iam-policy-binding ${FOURKEYS_PROJECT} --member="serviceAccount:${PARENT_PROJECTNUM}@cloudbuild.gserviceaccount.com" --role="roles/storage.admin" - -# launch container builds in background/parallel -gcloud builds submit ../event-handler --tag=gcr.io/${FOURKEYS_PROJECT}/event-handler --project=${PARENT_PROJECT} > event_handler.containerbuild.log & - -if [[ ! -z "$GIT_SYSTEM" ]]; then - gcloud builds submit ../bq-workers/${GIT_SYSTEM}-parser --tag=gcr.io/${FOURKEYS_PROJECT}/${GIT_SYSTEM}-parser --project=${PARENT_PROJECT} > ${GIT_SYSTEM}-parser.containerbuild.log & -fi - -if [[ ! -z "$CICD_SYSTEM" && "$CICD_SYSTEM" != "$GIT_SYSTEM" ]]; then - gcloud builds submit ../bq-workers/${CICD_SYSTEM}-parser --tag=gcr.io/${FOURKEYS_PROJECT}/${CICD_SYSTEM}-parser --project=${PARENT_PROJECT} > ${CICD_SYSTEM}-parser.containerbuild.log & -fi - -if [[ ! -z "$INCIDENT_SYSTEM" ]]; then - gcloud builds submit ../bq-workers/${INCIDENT_SYSTEM}-parser --tag=gcr.io/${FOURKEYS_PROJECT}/${INCIDENT_SYSTEM}-parser --project=${PARENT_PROJECT} > ${INCIDENT_SYSTEM}-parser.containerbuild.log & -fi - -# Dashboard image -gcloud builds submit ../dashboard --tag=gcr.io/${FOURKEYS_PROJECT}/fourkeys-grafana-dashboard --project=${PARENT_PROJECT} > fourkeys-grafana-dashboard.containerbuild.log & - -# wait for containers to be built, then continue -wait -echo "••••••••🔑••🔑••🔑••🔑••••••••" -echo "Invoking Terraform on project ${FOURKEYS_PROJECT}…" - -terraform apply --auto-approve - -echo "Terraform resource creation complete." -echo "••••••••🔑••🔑••🔑••🔑••••••••" - -if [ $GENERATE_DATA == "yes" ]; then - - TOKEN="" - - # Create an identity token if running in cloudbuild tests - if [[ "$(gcloud config get-value account)" == "${PARENT_PROJECTNUM}@cloudbuild.gserviceaccount.com" ]] - then - TOKEN=$(curl -X POST -H "content-type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d "{\"audience\": \"$(terraform output -raw event_handler_endpoint)\"}" \ - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/fourkeys@${FOURKEYS_PROJECT}.iam.gserviceaccount.com:generateIdToken" | \ - python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") - fi - - echo "generating data…" - WEBHOOK=$(terraform output -raw event_handler_endpoint) SECRET=$(terraform output -raw event_handler_secret) TOKEN=${TOKEN} python3 ../data-generator/generate_data.py --vc_system=${GIT_SYSTEM} -fi - -echo "••••••••🔑••🔑••🔑••🔑••••••••" -echo -e "Visit ${GREEN}$(terraform output -raw dashboard_endpoint)${NOCOLOR} to view your data in the dashboard template." - -if [[ ! -z "$CICD_SYSTEM" ]]; then - echo "••••••••🔑••🔑••🔑••🔑••••••••" - echo 'Setup complete! Run the following commands to get values needed to configure VCS webhook:' - echo -e "➡️ Webhook URL: ${GREEN}echo \$(terraform output -raw event_handler_endpoint)${NOCOLOR}" - echo -e "➡️ Secret: ${GREEN}echo \$(terraform output -raw event_handler_secret)${NOCOLOR}" -fi diff --git a/setup/main.tf b/setup/main.tf deleted file mode 100644 index 4015fc15..00000000 --- a/setup/main.tf +++ /dev/null @@ -1,19 +0,0 @@ -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = "4.18.0" - } - } -} - -resource "google_project_service" "run_api" { - project = var.google_project_id - service = "run.googleapis.com" -} - -resource "google_service_account" "fourkeys" { - account_id = "fourkeys" - display_name = "Service Account for Four Keys resources" -} diff --git a/setup/new_source.sh b/setup/new_source.sh deleted file mode 100644 index 483304d1..00000000 --- a/setup/new_source.sh +++ /dev/null @@ -1 +0,0 @@ -### TODO: #195 we need to make a new version of new_source.sh (see old version in "deprecated") for compatibility with the terraform installer \ No newline at end of file diff --git a/setup/outputs.tf b/setup/outputs.tf deleted file mode 100644 index c75b610e..00000000 --- a/setup/outputs.tf +++ /dev/null @@ -1,12 +0,0 @@ -output "event_handler_endpoint" { - value = google_cloud_run_service.event_handler.status[0]["url"] -} - -output "event_handler_secret" { - value = google_secret_manager_secret_version.event_handler.secret_data - sensitive = true -} - -output "dashboard_endpoint" { - value = "${module.dashboard_service.dashboard_endpoint}/d/yVtwoQ4nk/four-keys?orgId=1" -} diff --git a/setup/providers.tf b/setup/providers.tf deleted file mode 100644 index 8dea1690..00000000 --- a/setup/providers.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "google" { - project = var.google_project_id - region = var.google_region -} \ No newline at end of file diff --git a/setup/resource_bigquery.tf b/setup/resource_bigquery.tf deleted file mode 100644 index 8a27d561..00000000 --- a/setup/resource_bigquery.tf +++ /dev/null @@ -1,147 +0,0 @@ -resource "google_project_service" "bq_api" { - service = "bigquery.googleapis.com" - disable_dependent_services = true -} - -# The BigQuery API can take time to become interactive, so add a delay -# before attempting to create resources -resource "time_sleep" "wait_for_bq_api" { - depends_on = [ - google_project_service.bq_api - ] - - create_duration = "30s" # adjust this duration as needed -} - -resource "google_bigquery_dataset" "four_keys" { - dataset_id = "four_keys" - delete_contents_on_destroy = false - location = var.bigquery_region - access { - role = "OWNER" - special_group = "projectOwners" - } - access { - role = "WRITER" - user_by_email = google_service_account.fourkeys.email - } - depends_on = [ - time_sleep.wait_for_bq_api - ] -} - -resource "google_bigquery_table" "events_raw" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "events_raw" - schema = file("./events_raw_schema.json") - deletion_protection = false -} - -resource "google_bigquery_table" "events_enriched" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "events_enriched" - schema = file("./events_enriched_schema.json") - deletion_protection = false -} - -resource "google_bigquery_table" "view_events" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "events" - view { - query = file("../queries/events.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_bigquery_table.events_raw, - google_bigquery_table.events_enriched, - ] -} - -resource "google_bigquery_table" "view_changes" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "changes" - view { - query = file("../queries/changes.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_bigquery_table.events_raw - ] -} - -resource "google_bigquery_routine" "func_json2array" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - routine_id = "json2array" - routine_type = "SCALAR_FUNCTION" - return_type = "{\"typeKind\": \"ARRAY\", \"arrayElementType\": {\"typeKind\": \"STRING\"}}" - language = "JAVASCRIPT" - arguments { - name = "json" - data_type = "{\"typeKind\" : \"STRING\"}" - } - definition_body = file("../queries/function_json2array.js") -} - -resource "google_bigquery_routine" "func_multiFormatParseTimestamp" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - routine_id = "multiFormatParseTimestamp" - routine_type = "SCALAR_FUNCTION" - return_type = "{\"typeKind\" : \"TIMESTAMP\"}" - language = "SQL" - arguments { - name = "input" - data_type = "{\"typeKind\" : \"STRING\"}" - } - definition_body = file("../queries/function_multiFormatParseTimestamp.sql") -} - -resource "google_bigquery_table" "view_deployments" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "deployments" - view { - query = file("../queries/deployments.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_bigquery_table.events_raw, - google_bigquery_routine.func_json2array - ] -} - -resource "google_bigquery_table" "view_incidents" { - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "incidents" - view { - query = file("../queries/incidents.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_bigquery_table.events_raw, - google_bigquery_table.view_deployments, - google_bigquery_routine.func_multiFormatParseTimestamp - ] -} - -resource "google_project_iam_member" "parser_bq_project_access" { - project = google_service_account.fourkeys.project - role = "roles/bigquery.user" - member = "serviceAccount:${google_service_account.fourkeys.email}" -} - -resource "google_bigquery_dataset_iam_member" "parser_bq" { - project = google_service_account.fourkeys.project - dataset_id = google_bigquery_dataset.four_keys.dataset_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:${google_service_account.fourkeys.email}" -} - - -resource "google_project_iam_member" "parser_run_invoker" { - project = google_service_account.fourkeys.project - member = "serviceAccount:${google_service_account.fourkeys.email}" - role = "roles/run.invoker" -} \ No newline at end of file diff --git a/setup/resource_dashboard.tf b/setup/resource_dashboard.tf deleted file mode 100644 index cc8264db..00000000 --- a/setup/resource_dashboard.tf +++ /dev/null @@ -1,12 +0,0 @@ -module "dashboard_service" { - source = "./dashboard" - google_project_id = var.google_project_id - google_region = var.google_region - bigquery_region = var.bigquery_region - fourkeys_service_account_email = google_service_account.fourkeys.email - - depends_on = [ - google_project_service.run_api, - google_bigquery_dataset.four_keys - ] -} \ No newline at end of file diff --git a/setup/resource_event_handler.tf b/setup/resource_event_handler.tf deleted file mode 100644 index 7be0ad22..00000000 --- a/setup/resource_event_handler.tf +++ /dev/null @@ -1,71 +0,0 @@ -resource "google_project_service" "sm_api" { - service = "secretmanager.googleapis.com" -} - -resource "google_cloud_run_service" "event_handler" { - name = "event-handler" - location = var.google_region - - template { - spec { - containers { - image = "gcr.io/${var.google_project_id}/event-handler" - env { - name = "PROJECT_NAME" - value = var.google_project_id - } - } - service_account_name = google_service_account.fourkeys.email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - autogenerate_revision_name = true - - depends_on = [ - google_project_service.run_api, - google_bigquery_dataset.four_keys - ] - - metadata { - labels = { "created_by" : "fourkeys" } - } -} - -resource "google_cloud_run_service_iam_binding" "noauth" { - location = var.google_region - project = var.google_project_id - service = "event-handler" - - role = "roles/run.invoker" - members = ["allUsers"] - depends_on = [google_cloud_run_service.event_handler] -} - -resource "google_secret_manager_secret" "event_handler" { - secret_id = "event-handler" - replication { - automatic = true - } - depends_on = [google_project_service.sm_api] - labels = { "created_by" : "fourkeys" } -} - -resource "random_id" "event_handler_random_value" { - byte_length = "20" -} - -resource "google_secret_manager_secret_version" "event_handler" { - secret = google_secret_manager_secret.event_handler.id - secret_data = random_id.event_handler_random_value.hex -} - -resource "google_secret_manager_secret_iam_member" "event_handler" { - secret_id = google_secret_manager_secret.event_handler.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${google_service_account.fourkeys.email}" -} \ No newline at end of file diff --git a/setup/resource_parsers.tf b/setup/resource_parsers.tf deleted file mode 100644 index 8b81b9ff..00000000 --- a/setup/resource_parsers.tf +++ /dev/null @@ -1,13 +0,0 @@ -module "data_parser_service" { - for_each = toset(var.parsers) - source = "./data_parser" - parser_service_name = each.key - google_project_id = var.google_project_id - google_region = var.google_region - fourkeys_service_account_email = google_service_account.fourkeys.email - - depends_on = [ - google_project_service.run_api, - google_bigquery_dataset.four_keys - ] -} diff --git a/setup/variables.tf b/setup/variables.tf deleted file mode 100644 index 65aeb4b4..00000000 --- a/setup/variables.tf +++ /dev/null @@ -1,20 +0,0 @@ -variable "google_project_id" { - type = string -} - -variable "google_region" { - type = string -} - -variable "bigquery_region" { - type = string - validation { - condition = can(regex("^(US|EU)$", var.bigquery_region)) - error_message = "The value for 'bigquery_region' must be one of: 'US','EU'." - } -} - -variable "parsers" { - description = "list of data parsers to configure (e.g. 'gitlab','tekton')" - type = list(any) -} From dca3de6e8541de961b639c089180b369750ac90d Mon Sep 17 00:00:00 2001 From: Roger Martinez Date: Tue, 15 Nov 2022 14:20:19 -0800 Subject: [PATCH 21/21] rm old setup.sh --- setup/setup.sh | 155 ------------------------------------------------- 1 file changed, 155 deletions(-) delete mode 100755 setup/setup.sh diff --git a/setup/setup.sh b/setup/setup.sh deleted file mode 100755 index 3679ab62..00000000 --- a/setup/setup.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script configures installation variables, then invokes `install.sh` - -set -eEuo pipefail - -# PARSE INPUTS -CLEAN="false" -AUTO="false" -for i in "$@" -do - case $i in - -c | --clean ) CLEAN="true"; shift;; - -a | --auto ) AUTO="true"; shift;; - -h | --help ) echo "Usage: ./setup.sh [--clean] [--auto]"; exit 0; shift;; - *) ;; # unknown option - esac -done - -if [[ ${AUTO} == 'true' ]] -then - # populate setup variables (for use in testing/dev) - git_system_id=2 - cicd_system_id=1 - incident_system_id=1 - generate_mock_data=y - CLEAN='true' -else - printf "\n" - printf "Four Keys requires a Google Cloud project with billing enabled.\n" - printf "If you don't have a suitable project, exit this installer and create a project.\n" - - read -p "Enter the project ID for Four Keys installation (ex: 'my-project'): " FOURKEYS_PROJECT - read -p "Enter the region for Four Keys resources (ex: 'us-central1'): " FOURKEYS_REGION - read -p "Enter the location for Four Keys BigQuery resources ('US' or 'EU'): " BIGQUERY_REGION - - read -p "Which version control system are you using? - (1) GitLab - (2) GitHub - (3) Other - - Enter a selection (1 - 3): " git_system_id - - read -p " - Which CI/CD system are you using? - (1) Cloud Build - (2) Tekton - (3) GitLab - (4) CircleCI - (5) ArgoCD - (6) Other - - Enter a selection (1 - 6): " cicd_system_id - - read -p " - Which incident management system(s) are you using? - (1) PagerDuty - (2) Other - - Enter a selection (1 - 2): " incident_system_id - - printf "\n" - - if [[ ${git_system_id} == "1" ]] || [[ ${git_system_id} == "2" ]] - then - read -p "Would you like to generate mock data? (y/N): " generate_mock_data - generate_mock_data=${generate_mock_data:-no} - else - # offer mock data only in case of GitLab or GitHub - generate_mock_data="N" - fi -fi - -if [[ ${CLEAN} == 'true' ]] -then - # purge all local terraform state - rm -rf .terraform* *.containerbuild.log terraform.tfstate* terraform.tfvars -fi - -printf "\n" - -GIT_SYSTEM="" -CICD_SYSTEM="" -INCIDENT_SYSTEM="" -PAGERDUTY_SECRET="" - -case $git_system_id in - 1) GIT_SYSTEM="gitlab" ;; - 2) GIT_SYSTEM="github" ;; - *) echo "Please see the documentation to learn how to extend to VCS sources other than GitHub or GitLab" -esac - -case $cicd_system_id in - 1) CICD_SYSTEM="cloud-build" ;; - 2) CICD_SYSTEM="tekton" ;; - 3) CICD_SYSTEM="gitlab" ;; - 4) CICD_SYSTEM="circleci" ;; - 5) CICD_SYSTEM="argocd" ;; - *) echo "Please see the documentation to learn how to extend to CI/CD sources other than Cloud Build, Tekton, GitLab, CircleCI or GitHub." -esac - -case $incident_system_id in - 1) INCIDENT_SYSTEM="pagerduty"; read -p "Please enter the PagerDuty Signature Verification Token: " PAGERDUTY_SECRET ;; - *) echo "Please see the documentation to learn how to extend to incident sources other than PagerDuty." -esac - -if [ "$PAGERDUTY_SECRET" != "" ]; then - echo $PAGERDUTY_SECRET | tr -d '\n' | gcloud secrets create pager_duty_secret \ - --replication-policy=user-managed --locations ${FOURKEYS_REGION} \ - --data-file=- -fi - -if [ $generate_mock_data == "y" ]; then - GENERATE_DATA="yes" -else - GENERATE_DATA="no" -fi - -PARSERS="" -for PARSER in ${GIT_SYSTEM} ${CICD_SYSTEM} ${INCIDENT_SYSTEM}; do - if [ "${PARSERS}" == "" ]; then - PARSERS="\"${PARSER}\"" - else - PARSERS+=",\"${PARSER}\"" - fi -done - -# create a tfvars file -cat > terraform.tfvars </dev/null) -source install.sh \ No newline at end of file