Merge branch 'main' into fix-terraform-project

.github/workflows/codehealth.yaml

@@ -24,6 +24,9 @@ jobs:
  node-version: 20
  check-latest: true
 
+ - name: Use terraform
+ uses: hashicorp/setup-terraform@v3
+
  - name: Install node modules
  run: npm install
 
@@ -34,7 +37,7 @@ jobs:
  run: npm run eslint
 
  - name: Execute "npm run check-format"
- run: npm run check-format -- --log-level warn
+ run: npm run check-format
 
  - name: Execute "npm run mdlint"
  run: npm run mdlint
@@ -45,8 +48,5 @@ jobs:
  - name: Execute "npm run markdown-link-check"
  run: npm run markdown-link-check
 
- - name: Use terraform
- uses: hashicorp/setup-terraform@v3
-
  - name: terraform validate deployments
  run: npm run terraform-validate

configeditor/build-configeditor.sh

@@ -21,11 +21,26 @@ SCRIPTDIR=$(dirname "$0")
 cd "$SCRIPTDIR"
 
 npm install --quiet
-mkdir -p build/vanilla-jsoneditor
-[[ ! -e build/vanilla-jsoneditor/standalone.js ]] && \
- curl -o build/vanilla-jsoneditor/standalone.js \
- https://cdn.jsdelivr.net/npm/vanilla-jsoneditor@0.23.8/standalone.js
-
+JSONEDITOR_JS=build/vanilla-jsoneditor/standalone.js
+# renovate: datasource=npm packageName=vanilla-jsoneditor
+JSONEDITOR_VERSION=0.23.8
+JSONEDITOR_JS_URL="https://cdn.jsdelivr.net/npm/vanilla-jsoneditor@${JSONEDITOR_VERSION}/standalone.js"
+# sha256sum of file at $JSONEDITOR_JS_URL
+JSONEDITOR_JS_HASH="81886177f9cab8541f73e02aa195fcea27089acfdf5be48b20ed60f65543f6cf"
+if [[ ! -e "$JSONEDITOR_JS" ]]; then
+ echo "Downloading npm/vanilla-jsoneditor@${JSONEDITOR_VERSION}/standalone.js"
+ curl -s -o "$JSONEDITOR_JS" "$JSONEDITOR_JS_URL"
+
+ # Check sha256sum hash
+ if ! echo "$JSONEDITOR_JS_HASH $JSONEDITOR_JS" \
+ | sha256sum --check --quiet ; then
+ echo ""
+ echo "FAILED $JSONEDITOR_JS Checksum does not match expected value"
+ rm "$JSONEDITOR_JS"
+ exit 1
+ fi
+fi
+
 cp -r ../node_modules/js-yaml ../autoscaler-config.schema.json build/
 
 [[ "$1" == "--quiet" ]] || cat <<EOF

package.json

@@ -7,23 +7,26 @@
  "author": "Google Inc.",
  "main": "src/functions.js",
  "scripts": {
- "check-format": "prettier --check .",
+ "check-format": "npm run prettier-check && npm run terraform-fmt-check",
  "debug-poller-function": "node --inspect node_modules/@google-cloud/functions-framework --target=checkSpannerScaleMetricsHTTP",
  "debug-scaler-function": "node --inspect node_modules/@google-cloud/functions-framework --target=scaleSpannerInstanceHTTP --port=8081",
  "debug-forwarder-function": "node --inspect node_modules/@google-cloud/functions-framework --target=forwardFromHTTP",
  "eslint": "eslint .",
  "eslint-fix": "eslint --fix .",
  "format": "prettier --write .",
  "install-all": "npm install --save",
- "markdown-link-check": "find . -name '*.md' -not \\( -path './node_modules/*' -o -path './configeditor/build/*' -o -path '*/.terraform/*' -o -path './CHANGELOG.md' -prune \\) -print0 | xargs --null markdown-link-check --config markdown-link-checker.json --quiet",
+ "markdown-link-check": "find . -name '*.md' -not \\( -path './node_modules/*' -o -path './configeditor/build/*' -o -path '*/.terraform/*' -o -path './CHANGELOG.md' -prune \\) -print0 | xargs -0 markdown-link-check --config markdown-link-checker.json --quiet",
  "mdlint": "markdownlint '**/*.md' --config .mdl.json --ignore '**/node_modules/**' --ignore '**/build/**' --ignore 'code-of-conduct.md' --ignore 'CHANGELOG.md'",
  "poller-job": "node -e \"require('./src/poller/index').main()\"",
  "prepare": "{ git rev-parse --is-inside-work-tree >/dev/null 2>/dev/null && test \"$NODE_ENV\" != production -a \"$CI\" != true && husky ; } || true",
  "prettier": "prettier --write .",
+ "prettier-check": "prettier --check .",
  "scaler-service": "node -e \"require('./src/scaler/index').main()\"",
  "start-forwarder-function": "functions-framework --target=forwardFromHTTP",
  "start-poller-function": "functions-framework --target=checkSpannerScaleMetricsHTTP",
  "start-scaler-function": "functions-framework --target=scaleSpannerInstanceHTTP --port=8081",
+ "terraform-fmt": "echo 'Running Terraform fmt'; find . -name '*.tf' -print0 | xargs -0 terraform fmt",
+ "terraform-fmt-check": "echo 'Checking Terraform format'; if ! find . -name '*.tf' -print0 | xargs -0 terraform fmt -check; then echo 'Files need reformatting: npm run terraform-fmt'; exit 1; fi",
  "terraform-validate": "set -e ; package_root=\"$(pwd)\"; for x in gke/decoupled gke/unified cloud-functions/distributed/app-project cloud-functions/distributed/autoscaler-project cloud-functions/per-project ; do cd \"$package_root/terraform/$x\" ; echo \"\n\nValidating Terraform in $PWD\n\n\" ; terraform init ; terraform validate; done",
  "test": "NODE_ENV=test nyc --reporter=text mocha --recursive src --extension=.test.js",
  "test-e2e": "pushd terraform/cloud-functions/per-project/test && go test -run . -timeout 60m --tags=e2e && popd",

renovate.json5

@@ -43,5 +43,19 @@
  groupName: "docker-containers",
  },
  ],
+ customManagers: [
+ {
+ customType: "regex",
+ description: "Update _VERSION variables in Dockerfiles, shell scripts",
+ fileMatch: [
+ "(^|/|\\.)([Dd]ocker|[Cc]ontainer)file$",
+ "(^|/)([Dd]ocker|[Cc]ontainer)file[^/]*$",
+ "(^|/)*.sh",
+ ],
+ matchStrings: [
+ '# renovate: datasource=(?<datasource>[a-z-]+?)(?: depName=(?<depName>.+?))? packageName=(?<packageName>.+?)(?: versioning=(?<versioning>[a-z-]+?))?\\s(?:ENV|ARG)?\\s*.+?_VERSION="?(?<currentValue>.+?)"?\\s',
+ ],
+ },
+ ],
  rangeStrategy: "bump",
 }

terraform/gke/decoupled/main.tf

@@ -21,7 +21,7 @@ terraform {
  version = ">= 6.3.0"
  }
  kubernetes = {
- source = "hashicorp/kubernetes"
+ source  = "hashicorp/kubernetes"
  version = ">= 2.32.0"
  }
  }

terraform/gke/unified/main.tf

@@ -21,7 +21,7 @@ terraform {
  version = ">= 6.3.0"
  }
  kubernetes = {
- source = "hashicorp/kubernetes"
+ source  = "hashicorp/kubernetes"
  version = "~> 2.32.0"
  }
  }

terraform/modules/autoscaler-base/outputs.tf

@@ -17,10 +17,10 @@
 output "build_sa_id" {
  value = google_service_account.build_sa.id
  description = "Service account ID for Builder SA"
- depends_on = [ google_project_iam_binding.build_iam ]
+ depends_on  = [google_project_iam_binding.build_iam]
 }
 output "build_sa_email" {
  value = google_service_account.build_sa.email
  description = "Service account email for Builder SA"
- depends_on = [ google_project_iam_binding.build_iam ]
+ depends_on  = [google_project_iam_binding.build_iam]
 }

terraform/modules/autoscaler-cluster/main.tf

@@ -131,9 +131,9 @@ resource "kubernetes_namespace" "autoscaler_namespace" {
 }
 
 module "workload_identity_poller" {
- count  = var.unified_components ? 0 : 1
- source  = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
- version  = ">= 33.0.4"
+ count = var.unified_components ? 0 : 1
+ source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
+ version = ">= 33.0.4"
 
  project_id = var.project_id
  namespace = "spanner-autoscaler"
@@ -144,8 +144,8 @@ module "workload_identity_poller" {
 }
 
 module "workload_identity_scaler" {
- source  = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
- version  = ">= 33.0.4"
+ source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
+ version = ">= 33.0.4"
 
  project_id = var.project_id
  namespace = "spanner-autoscaler"
@@ -157,8 +157,8 @@ module "workload_identity_scaler" {
 
 
 module "workload_identity_otel_collector" {
- source  = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
- version  = ">= 33.0.4"
+ source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
+ version = ">= 33.0.4"
 
  project_id = var.project_id
  namespace = "spanner-autoscaler"
@@ -170,8 +170,8 @@ module "workload_identity_otel_collector" {
 
 
 module "cluster" {
- source  = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster"
- version  = ">= 33.0.4"
+ source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster"
+ version = ">= 33.0.4"
 
  project_id = var.project_id
  name = var.name

terraform/modules/autoscaler-cluster/variables.tf

@@ -70,9 +70,9 @@ variable "scaler_sa_email" {
 }
 
 variable "otel_collector_sa_name" {
- type = string
+ type  = string
  description = "The name of the service account and workload identity to be created and used by the OpenTelemetry Collector workload"
- default = "otel-collector-sa"
+ default  = "otel-collector-sa"
 }
 
 variable "unified_components" {

terraform/modules/autoscaler-functions/main.tf

@@ -77,7 +77,17 @@ data "archive_file" "local_source" {
  type = "zip"
  source_dir = abspath("${path.module}/../../..")
  output_path = "${var.local_output_path}/src.zip"
- excludes = [".git", ".github", ".nyc_output", ".vscode", "kubernetes", "node_modules", "resources", "scaler", "terraform"]
+ excludes = [
+ ".git",
+ ".github",
+ ".nyc_output",
+ ".vscode",
+ "kubernetes",
+ "node_modules",
+ "resources",
+ "scaler",
+ "terraform"
+ ]
 }
 
 resource "google_storage_bucket_object" "gcs_functions_source" {

terraform/modules/monitoring/main.tf

@@ -15,7 +15,7 @@
  */
 
 resource "google_monitoring_dashboard" "dashboard" {
- project  = var.project_id
+ project = var.project_id
  dashboard_json = templatefile("${path.module}/dashboard.json.tftpl", {
  thresholds_high_priority_cpu_percentage = var.dashboard_threshold_high_priority_cpu_percentage
  thresholds_rolling_24hr_cpu_percentage = var.dashboard_threshold_rolling_24_hr_percentage

terraform/modules/scheduler/main.tf

@@ -16,26 +16,26 @@
 
 locals {
  config = var.json_config != "" ? var.json_config : base64encode(jsonencode([
-  merge ({
-  "projectId" : "${var.project_id}",
-  "instanceId" : "${var.spanner_name}",
-  "scalerPubSubTopic" : "${var.target_pubsub_topic}",
-  "units" : "${var.units}",
-  "minSize" : var.min_size,
-  "maxSize" : var.max_size,
-  "scalingMethod" : "${var.scaling_method}",
-  "stateDatabase": var.terraform_spanner_state ? {
-  "name":  "spanner",
-  "instanceId": "${var.spanner_state_name}"
-  "databaseId": "spanner-autoscaler-state"
+ merge({
+ "projectId" : "${var.project_id}",
+ "instanceId" : "${var.spanner_name}",
+ "scalerPubSubTopic" : "${var.target_pubsub_topic}",
+ "units" : "${var.units}",
+ "minSize" : var.min_size,
+ "maxSize" : var.max_size,
+ "scalingMethod" : "${var.scaling_method}",
+ "stateDatabase" : var.terraform_spanner_state ? {
+ "name" : "spanner",
+ "instanceId" : "${var.spanner_state_name}"
+ "databaseId" : "spanner-autoscaler-state"
  } : {
-  "name":  "firestore",
-  }
+ "name" : "firestore",
+ }
  },
  var.state_project_id != null ? {
  "stateProjectId" : "${var.state_project_id}"
-  } : {})
-  ]))
+ } : {})
+ ]))
 }
 
 resource "google_app_engine_application" "app" {