Merge branch 'master' of https://github.com/AkshayKalbhor/autoscaler

.gitignore

@@ -42,3 +42,6 @@ terraform/*/*/*.json
 # Kubernetes manifests generated from templates
 kubernetes/autoscaler-config/*.yaml
 kubernetes/**/resourcegroup.yaml
+
+# Terratest
+.test-data

terraform/cloud-functions/distributed/README.md

@@ -294,15 +294,15 @@ topic and function in the project where the Spanner instances live.
  will create:
 
  ```sh
- export TF_VAR_state_spanner_name=<INSERT_STATE_SPANNER_INSTANCE_NAME>
+ export TF_VAR_spanner_state_name=<INSERT_STATE_SPANNER_INSTANCE_NAME>
  ```
 
  If you already have an Spanner instance where state must be stored,
  only set the project id and the name of your instance:
 
  ```sh
  export TF_VAR_state_project_id=<INSERT_YOUR_STATE_PROJECT_ID>
- export TF_VAR_state_spanner_name=<INSERT_YOUR_STATE_SPANNER_INSTANCE_NAME>
+ export TF_VAR_spanner_state_name=<INSERT_YOUR_STATE_SPANNER_INSTANCE_NAME>
  ```
 
  In your own instance, make sure you create the the database

terraform/cloud-functions/distributed/app-project/main.tf

@@ -40,13 +40,15 @@ data "terraform_remote_state" "autoscaler" {
 module "spanner" {
  source = "../../../modules/spanner"
 
- terraform_spanner_state = var.terraform_spanner_state
- terraform_spanner_test = var.terraform_spanner_test
- project_id = var.project_id
- spanner_name = var.spanner_name
- state_spanner_name = var.state_spanner_name
- poller_sa_email = data.terraform_remote_state.autoscaler.outputs.poller_sa_email
- scaler_sa_email = data.terraform_remote_state.autoscaler.outputs.scaler_sa_email
+ terraform_spanner_state = var.terraform_spanner_state
+ terraform_spanner_test = var.terraform_spanner_test
+ project_id = var.project_id
+ spanner_name = var.spanner_name
+ spanner_state_name = var.spanner_state_name
+ spanner_test_processing_units = var.spanner_test_processing_units
+ spanner_state_processing_units = var.spanner_state_processing_units
+ poller_sa_email = data.terraform_remote_state.autoscaler.outputs.poller_sa_email
+ scaler_sa_email = data.terraform_remote_state.autoscaler.outputs.scaler_sa_email
 }
 
 module "forwarder" {
@@ -66,7 +68,7 @@ module "scheduler" {
  target_pubsub_topic = data.terraform_remote_state.autoscaler.outputs.scaler_topic
  terraform_spanner_state = var.terraform_spanner_state
  state_project_id = var.state_project_id
- state_spanner_name = var.state_spanner_name
+ spanner_state_name = var.spanner_state_name
 }
 
 module "monitoring" {

terraform/cloud-functions/distributed/app-project/variables.tf

@@ -33,7 +33,7 @@ variable "spanner_name" {
  default = "autoscale-test"
 }
 
-variable "state_spanner_name" {
+variable "spanner_state_name" {
  type = string
  default = "autoscale-test-state"
 }
@@ -50,6 +50,16 @@ variable "terraform_spanner_state" {
  default = false
 }
 
+variable "spanner_test_processing_units" {
+ description = "Default processing units for test Spanner, if created"
+ default = 100
+}
+
+variable "spanner_state_processing_units" {
+ description = "Default processing units for state Spanner, if created"
+ default = 100
+}
+
 variable "state_project_id" {
  type = string
 }

terraform/cloud-functions/per-project/README.md

@@ -205,14 +205,14 @@ In this section you prepare your project for deployment.
  will create:
 
  ```sh
- export TF_VAR_state_spanner_name=<INSERT_STATE_SPANNER_INSTANCE_NAME>
+ export TF_VAR_spanner_state_name=<INSERT_STATE_SPANNER_INSTANCE_NAME>
  ```
 
  If you already have a Spanner instance where state must be stored,
  only set the the name of your instance.
 
  ```sh
- export TF_VAR_state_spanner_name=<INSERT_YOUR_STATE_SPANNER_INSTANCE_NAME>
+ export TF_VAR_spanner_state_name=<INSERT_YOUR_STATE_SPANNER_INSTANCE_NAME>
  ```
 
  In your own instance, make sure you create the the database

terraform/cloud-functions/per-project/main.tf

@@ -54,13 +54,15 @@ module "firestore" {
 module "spanner" {
  source = "../../modules/spanner"
 
- terraform_spanner_state = var.terraform_spanner_state
- terraform_spanner_test = var.terraform_spanner_test
- project_id = local.app_project_id
- spanner_name = var.spanner_name
- poller_sa_email = module.autoscaler-base.poller_sa_email
- scaler_sa_email = module.autoscaler-base.scaler_sa_email
- state_spanner_name = var.state_spanner_name
+ terraform_spanner_state = var.terraform_spanner_state
+ terraform_spanner_test = var.terraform_spanner_test
+ project_id = local.app_project_id
+ spanner_name = var.spanner_name
+ spanner_state_name = var.spanner_state_name
+ spanner_test_processing_units = var.spanner_test_processing_units
+ spanner_state_processing_units = var.spanner_state_processing_units
+ poller_sa_email = module.autoscaler-base.poller_sa_email
+ scaler_sa_email = module.autoscaler-base.scaler_sa_email
 }
 
 module "scheduler" {
@@ -71,7 +73,7 @@ module "scheduler" {
  pubsub_topic = module.autoscaler-functions.poller_topic
  target_pubsub_topic = module.autoscaler-functions.scaler_topic
  terraform_spanner_state = var.terraform_spanner_state
- state_spanner_name = var.state_spanner_name
+ spanner_state_name = var.spanner_state_name
 
  // Example of passing config as json
  // json_config = base64encode(jsonencode([{

terraform/cloud-functions/per-project/outputs.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "scheduler_job_id" {
+ value = module.scheduler.scheduler_job_id
+ description = "ID of the Scheduler job"
+}

terraform/cloud-functions/per-project/test/Makefile

@@ -0,0 +1,23 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+test-e2e: ## Run the end-to-end test
+ @echo "Running E2E test..."
+ @go test -v -run ./ -timeout 30m --tags=e2e
+
+.PHONY: help
+.DEFAULT_GOAL := help
+
+help:
+ @fgrep -h "##" $(MAKEFILE_LIST) | sed -e 's/\(\:.*\#\#\)/\:\ /' | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'

terraform/cloud-functions/per-project/test/go.mod

@@ -0,0 +1,104 @@
+module github.com/cloudspannerecosystem/autoscaler/terraform/cloud-functions/per-project
+
+go 1.21
+
+require (
+ cloud.google.com/go/scheduler v1.10.0
+ cloud.google.com/go/spanner v1.46.0
+ github.com/gruntwork-io/terratest v0.43.0
+ github.com/sethvargo/go-envconfig v0.9.0
+ github.com/stretchr/testify v1.8.4
+ google.golang.org/protobuf v1.30.0
+)
+
+require (
+ cloud.google.com/go v0.110.0 // indirect
+ cloud.google.com/go/compute v1.19.3 // indirect
+ cloud.google.com/go/compute/metadata v0.2.3 // indirect
+ cloud.google.com/go/iam v0.13.0 // indirect
+ cloud.google.com/go/longrunning v0.4.1 // indirect
+ cloud.google.com/go/storage v1.28.1 // indirect
+ github.com/agext/levenshtein v1.2.3 // indirect
+ github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+ github.com/aws/aws-sdk-go v1.44.122 // indirect
+ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+ github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+ github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect
+ github.com/go-logr/logr v1.2.3 // indirect
+ github.com/go-openapi/jsonpointer v0.19.6 // indirect
+ github.com/go-openapi/jsonreference v0.20.1 // indirect
+ github.com/go-openapi/swag v0.22.3 // indirect
+ github.com/go-sql-driver/mysql v1.4.1 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.3 // indirect
+ github.com/google/gnostic v0.5.7-v3refs // indirect
+ github.com/google/go-cmp v0.5.9 // indirect
+ github.com/google/gofuzz v1.1.0 // indirect
+ github.com/google/s2a-go v0.1.4 // indirect
+ github.com/google/uuid v1.3.0 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+ github.com/googleapis/gax-go/v2 v2.10.0 // indirect
+ github.com/gruntwork-io/go-commons v0.8.0 // indirect
+ github.com/hashicorp/errwrap v1.0.0 // indirect
+ github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+ github.com/hashicorp/go-getter v1.7.1 // indirect
+ github.com/hashicorp/go-multierror v1.1.0 // indirect
+ github.com/hashicorp/go-safetemp v1.0.0 // indirect
+ github.com/hashicorp/go-version v1.6.0 // indirect
+ github.com/hashicorp/hcl/v2 v2.9.1 // indirect
+ github.com/hashicorp/terraform-json v0.13.0 // indirect
+ github.com/imdario/mergo v0.3.11 // indirect
+ github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/compress v1.15.11 // indirect
+ github.com/mailru/easyjson v0.7.7 // indirect
+ github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
+ github.com/mitchellh/go-homedir v1.1.0 // indirect
+ github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+ github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+ github.com/moby/spdystream v0.2.0 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/pquerna/otp v1.2.0 // indirect
+ github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/spf13/pflag v1.0.5 // indirect
+ github.com/tmccombs/hcl2json v0.3.3 // indirect
+ github.com/ulikunitz/xz v0.5.10 // indirect
+ github.com/urfave/cli v1.22.2 // indirect
+ github.com/zclconf/go-cty v1.9.1 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ golang.org/x/crypto v0.9.0 // indirect
+ golang.org/x/net v0.10.0 // indirect
+ golang.org/x/oauth2 v0.8.0 // indirect
+ golang.org/x/sys v0.8.0 // indirect
+ golang.org/x/term v0.8.0 // indirect
+ golang.org/x/text v0.9.0 // indirect
+ golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+ google.golang.org/api v0.125.0 // indirect
+ google.golang.org/appengine v1.6.7 // indirect
+ google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+ google.golang.org/grpc v1.55.0 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ k8s.io/api v0.27.2 // indirect
+ k8s.io/apimachinery v0.27.2 // indirect
+ k8s.io/client-go v0.27.2 // indirect
+ k8s.io/klog/v2 v2.90.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+ k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+ sigs.k8s.io/yaml v1.3.0 // indirect
+)