Merge pull request #4415 from consideRatio/pr/gcp-terraform-budgets

terraform, gcp: use dynamic budgets based on last month
2i2c-org · Jul 11, 2024 · 61d84bf · 61d84bf
2 parents 3a04c42 + f868236
commit 61d84bf
Show file tree

Hide file tree

Showing 14 changed files with 32 additions and 70 deletions.
diff --git a/docs/howto/budget-alerts.md b/docs/howto/budget-alerts.md
@@ -29,11 +29,5 @@ Then edit the following variables in the relevant `.tfvars` file for the cluster
  This will open a pane that gives you the Billing Account ID.
  - For accounts that we don't manage, the process is the same but _we may not have permission to view the Billing Account ID_.
  In this case, we cannot enable budget alerting for this project.
-- **Set `budget_alert_amount`.**
- Current practice is to set this to the average expenditure of the last 3 months, plus 20%.
- You can find values to calculate that in the [Billing Reports console](https://console.cloud.google.com/billing/0157F7-E3EA8C-25AC3C/reports?organizationId=184174754493&project=two-eye-two-see).
- _Make sure you select only the project you are interested in from the Projects field in the Filters pane on the right side of the screen._
- - If you are setting this up for a new cluster, you obviously don't have this information yet!
- Instead, set the value to `400` and we can adjust as the community begins to use it.
 
 With these variables set, you are ready to open a PR and perform a terraform apply!
diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf
@@ -1,61 +1,60 @@
-# Alerts sent to support@2i2c.org for things that *will go bad* in the future
-# if left unattended. Should *not* be used for immediate outages
+# Alerts sent to support+budget-${var.prefix}@2i2c.org for things that *will go
+# bad* in the future if left unattended. Should *not* be used for immediate
+# outages
+
+data "google_project" "project" {
+ project_id = var.project_id
+}
 
 resource "google_monitoring_notification_channel" "support_email" {
  count = var.budget_alert_enabled ? 1 : 0
  project = var.project_id
- display_name = "support@2i2c.org email"
+ display_name = "Email support+budget-${var.prefix}@2i2c.org"
  type = "email"
  labels = {
- email_address = "support@2i2c.org"
+ email_address = "support+budget-${var.prefix}@2i2c.org"
  }
 }
 
-data "google_project" "project" {
- project_id = var.project_id
-}
-
 # Need to explicitly enable https://console.cloud.google.com/apis/library/billingbudgets.googleapis.com?project=two-eye-two-see
-resource "google_billing_budget" "budget" {
+# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/billing_budget
+resource "google_billing_budget" "budgets" {
  count = var.budget_alert_enabled ? 1 : 0
 
  billing_account = var.billing_account_id
- display_name = "Billing alert"
+ display_name = "Auto-adjusting budget for ${var.prefix}"
+
+ all_updates_rule {
+ monitoring_notification_channels = [
+ google_monitoring_notification_channel.support_email[0].id,
+ ]
+ disable_default_iam_recipients = true
+ }
 
  budget_filter {
  # Use project number here, as project_name seems to be converted internally to number
  # If we don't do this, `terraform apply` is not clean
  # This is a bug in the google provider / budgets API https://github.com/hashicorp/terraform-provider-google/issues/8444
  projects = ["projects/${data.google_project.project.number}"]
  credit_types_treatment = "INCLUDE_ALL_CREDITS"
+ calendar_period = "MONTH"
  }
 
  amount {
- specified_amount {
- currency_code = var.budget_alert_currency
- units = var.budget_alert_amount
- }
+ last_period_amount = true
  }
 
- all_updates_rule {
- monitoring_notification_channels = [
- google_monitoring_notification_channel.support_email[0].id,
- ]
- disable_default_iam_recipients = true
- }
  # NOTE: These threshold_rules *MUST BE ORDERED BY threshold_percent* in ascending order!
  # If not, we'll run into https://github.com/hashicorp/terraform-provider-google/issues/8444
  # and terraform apply won't be clean.
  threshold_rules {
- # Alert when *actual* spend reached 80% of budget
- threshold_percent = 1.0
+ # Alert when *actual* spend reached 120% of budget (last month's spend)
+ threshold_percent = 1.2
  spend_basis = "CURRENT_SPEND"
  }
  threshold_rules {
- # Alert when *forecasted* spend is about to blow over our budget
- # Adding the extra 1% to help terraform not redo this each time.
- threshold_percent = 1.01
+ # Alert when *forecasted* spend reached 120% of budget (last month's spend)
+ threshold_percent = 1.2
  spend_basis = "FORECASTED_SPEND"
  }
-
-}
+}
diff --git a/terraform/gcp/projects/2i2c-uk.tfvars b/terraform/gcp/projects/2i2c-uk.tfvars
@@ -4,9 +4,7 @@ project_id = "two-eye-two-see-uk"
 zone = "europe-west2-b"
 region = "europe-west2"
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "830"
-billing_account_id = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
  min_master_version : "1.29.1-gke.1589018",

diff --git a/terraform/gcp/projects/awi-ciroh-2.tfvars b/terraform/gcp/projects/awi-ciroh-2.tfvars
@@ -10,9 +10,8 @@ filestores = {
  "filestore" = { capacity_gb = 5939 }
 }
 
-# This project does not have cloud costs passed through by 2i2c
+# Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount = ""
 billing_account_id = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/awi-ciroh.tfvars b/terraform/gcp/projects/awi-ciroh.tfvars
@@ -1,6 +1,5 @@
 budget_alert_enabled = false
 billing_account_id = "0157F7-E3EA8C-25AC3C"
-budget_alert_amount = "800"
 
 prefix = "awi-ciroh"
 project_id = "awi-ciroh"

diff --git a/terraform/gcp/projects/catalystproject-latam.tfvars b/terraform/gcp/projects/catalystproject-latam.tfvars
@@ -4,9 +4,7 @@ region = "southamerica-east1"
 zone = "southamerica-east1-c"
 enable_network_policy = true
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "1672"
-billing_account_id = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
  min_master_version : "1.29.1-gke.1589018",

diff --git a/terraform/gcp/projects/cloudbank.tfvars b/terraform/gcp/projects/cloudbank.tfvars
@@ -5,10 +5,9 @@ zone = "us-central1-b"
 region = "us-central1"
 regional_cluster = false
 
-# We don't have enough access to enable this
+# Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
 billing_account_id = ""
-budget_alert_amount = ""
 
 k8s_versions = {
  # NOTE: This isn't a regional cluster / highly available cluster, when

diff --git a/terraform/gcp/projects/cluster.tfvars.template b/terraform/gcp/projects/cluster.tfvars.template
@@ -13,7 +13,6 @@ region = "{{ cluster_region }}"
 
 # Config required to enable automatic budget alerts to be sent to support@2i2c.org
 budget_alert_enabled = false
-budget_alert_amount = ""
 billing_account_id = ""
 
 # TODO: Before applying this, identify a k8s version to specify. Pick the latest

diff --git a/terraform/gcp/projects/hhmi.tfvars b/terraform/gcp/projects/hhmi.tfvars
@@ -6,9 +6,7 @@ region = "us-west2"
 
 core_node_machine_type = "n2-highmem-4"
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "797"
-billing_account_id = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
  min_master_version : "1.29.1-gke.1589020",

diff --git a/terraform/gcp/projects/leap.tfvars b/terraform/gcp/projects/leap.tfvars
@@ -6,7 +6,6 @@ core_node_machine_type = "n2-highmem-4"
 
 # Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount = ""
 billing_account_id = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/linked-earth.tfvars b/terraform/gcp/projects/linked-earth.tfvars
@@ -6,7 +6,6 @@ region = "us-central1"
 
 # Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount = ""
 billing_account_id = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/pangeo-hubs.tfvars b/terraform/gcp/projects/pangeo-hubs.tfvars
@@ -29,7 +29,6 @@ enable_logging = false
 
 # We don't have enough rights to make billing alerts
 budget_alert_enabled = false
-budget_alert_amount = ""
 billing_account_id = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars
@@ -5,9 +5,7 @@ zone = "us-central1-b"
 region = "us-central1"
 regional_cluster = false
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "1880"
-billing_account_id = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
  # NOTE: This isn't a regional cluster / highly available cluster, when

diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf
@@ -31,22 +31,6 @@ variable "billing_account_id" {
  EOT
 }
 
-variable "budget_alert_currency" {
- type = string
- default = "USD"
- description = <<-EOT
- Currency used for budget alerts.
- EOT
-}
-
-variable "budget_alert_amount" {
- type = string
- description = <<-EOT
- Amount of *forecasted spend* at which to send a billing alert. Current practice
- is to set this to the average of the last 3 months expenditure + 20%.
- EOT
-}
-
 variable "budget_alert_enabled" {
  type = bool
  default = true