diff --git a/.github/actions/debug-output/action.yaml b/.github/actions/debug-output/action.yaml index 1fe4ae4bf..334c3f7a6 100644 --- a/.github/actions/debug-output/action.yaml +++ b/.github/actions/debug-output/action.yaml @@ -22,6 +22,6 @@ runs: uds zarf tools kubectl get events -A --sort-by='.lastTimestamp' | tee /tmp/debug-k-get-events.log || true echo "::endgroup::" echo "::group::kubectl describe nodes" - uds zarf tools kubectl describe nodes k3d-uds-server-0 | tee /tmp/debug-k-describe-node.log || true + uds zarf tools kubectl describe nodes | tee /tmp/debug-k-describe-node.log || true echo "::endgroup::" shell: bash diff --git a/.github/bundles/aks/uds-bundle.yaml b/.github/bundles/aks/uds-bundle.yaml new file mode 100644 index 000000000..7887dd9db --- /dev/null +++ b/.github/bundles/aks/uds-bundle.yaml @@ -0,0 +1,137 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +kind: UDSBundle +metadata: + name: uds-core-aks-nightly + description: A UDS bundle for deploying UDS Core on AKS + version: "0.29.0" + +packages: + - name: init + repository: ghcr.io/zarf-dev/packages/init + ref: v0.41.0 + + - name: core + path: ../../../build + # x-release-please-start-version + ref: 0.29.0 + # x-release-please-end + overrides: + istio-admin-gateway: + gateway: + values: + - path: service.annotations + value: + service.beta.kubernetes.io/azure-load-balancer-internal: "false" + service.beta.kubernetes.io/azure-load-balancer-sku: "Standard" + service.beta.kubernetes.io/azure-load-balancer-resource-group: "${NODE_RESOURCE_GROUP_NAME}" + + istio-tenant-gateway: + gateway: + values: + - path: service.annotations + value: + service.beta.kubernetes.io/azure-load-balancer-internal: "false" + service.beta.kubernetes.io/azure-load-balancer-sku: "Standard" + service.beta.kubernetes.io/azure-load-balancer-resource-group: "${NODE_RESOURCE_GROUP_NAME}" + loki: + loki: + variables: + - name: AZURE_LOKI_STORAGE_ACCOUNT + description: "Name of the Storage Account to use for storing logs" + path: "loki.storage_config.azure.account_name" + - name: AZURE_LOKI_STORAGE_ACCOUNT_ACCESS_KEY + description: "Primary access Key for the Storage Account" + path: "loki.storage_config.azure.account_key" + - name: AZURE_LOKI_STORAGE_ACCOUNT_CONTAINER + description: "The destination container in the Storage Account where logs will be saved" + path: "loki.storage_config.azure.container_name" + values: + - path: loki.storage.type + value: "azure" + + kube-prometheus-stack: + kube-prometheus-stack: + values: + - path: kube-state-metrics + value: + resources: + limits: + memory: 512Mi + grafana: + grafana: + variables: + - name: GRAFANA_HA + description: Enable HA Grafana + path: autoscaling.enabled + uds-grafana-config: + variables: + - name: GRAFANA_PG_HOST + description: Grafana postgresql host + path: postgresql.host + - name: GRAFANA_PG_PORT + description: Grafana postgresql port + path: postgresql.port + - name: GRAFANA_PG_DATABASE + description: Grafana postgresql database + path: postgresql.database + - name: GRAFANA_PG_PASSWORD + description: Grafana postgresql password + path: postgresql.password + - name: GRAFANA_PG_USER + description: Grafana postgresql username + path: postgresql.user + + neuvector: + core: + values: + - path: runtimePath + value: /run/containerd/containerd.sock + - path: enforcer.tolerations + value: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: dedicated + operator: Exists + velero: + velero: + variables: + - name: AZURE_VELERO_STORAGE_ACCOUNT + description: "Name of the Storage Account to use for storing backups" + path: "configuration.backupStorageLocation[0].config.storageAccount" + - name: AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY + description: "Primary access Key for the Storage Account" + path: "configuration.backupStorageLocation[0].config.storageAccountKeyEnvVar" + - name: AZURE_VELERO_STORAGE_ACCOUNT_CONTAINER + description: "The destination container in the Storage Account where backups will be saved" + path: "configuration.backupStorageLocation[0].bucket" + - name: AZURE_RESOURCE_GROUP + description: "The name of the resource group that the Storage Account is in" + path: "configuration.backupStorageLocation[0].config.resourceGroup" + - name: AZURE_SUBSCRIPTION_ID + description: "The resource ID of the Azure Subscription that is being used" + path: "configuration.backupStorageLocation[0].config.subscriptionId" + - name: VELERO_CLIENT_SECRET_ENV_VAR + description: "Name of the env variable that velero will use to read Azure config" + path: "configuration.backupStorageLocation[0].config.storageAccountKeyEnvVar" + default: "AZURE_STORAGE_ACCOUNT_ACCESS_KEY" + - name: VELERO_BACKUP_STORAGE_CONFIG_NAME + description: "Name of the Backup Storage Location" + path: "configuration.backupStorageLocation[0].name" + default: "default" + - name: VELERO_STORAGE_PROVIDER + description: "Type of storage provider that will be used" + path: "configuration.backupStorageLocation[0].provider" + default: "azure" + values: + - path: credentials + value: + useSecret: true + secretContents: + cloud: | + AZURE_STORAGE_ACCOUNT_ACCESS_KEY=${AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY} + AZURE_CLOUD_NAME=AzurePublicCloud diff --git a/.github/bundles/aks/uds-config.yaml b/.github/bundles/aks/uds-config.yaml new file mode 100644 index 000000000..4ab2f6d06 --- /dev/null +++ b/.github/bundles/aks/uds-config.yaml @@ -0,0 +1,14 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +# Overwritten in ci by uds-config.tf in test-infra/azure/aks +variables: + core: + azure_loki_storage_account: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT} + azure_loki_storage_account_access_key: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT_ACCESS_KEY} + azure_loki_storage_account_container: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT_CONTAINER} + azure_velero_storage_account: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT} + azure_velero_storage_account_acces_key: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY} + azure_velero_storage_account_container: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT_CONTAINER} + azure_subscription_id: ${ZARF_VAR_AZURE_SUBSCRIPTION_ID} + azure_resource_group: ${ZARF_VAR_AZURE_RESOURCE_GROUP} diff --git a/.github/test-infra/aws/eks/main.tf b/.github/test-infra/aws/eks/main.tf index 07d5821a5..aee6089bf 100644 --- a/.github/test-infra/aws/eks/main.tf +++ b/.github/test-infra/aws/eks/main.tf @@ -1,7 +1,6 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial - resource "random_id" "default" { byte_length = 2 } diff --git a/.github/test-infra/aws/eks/uds-config.tf b/.github/test-infra/aws/eks/uds-config.tf new file mode 100644 index 000000000..29f1963c8 --- /dev/null +++ b/.github/test-infra/aws/eks/uds-config.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +resource "local_sensitive_file" "uds_config" { + filename = "../../../bundles/eks/uds-config.yaml" + content = yamlencode({ + "options" : { + "architecture" : "amd64" + }, + "variables" : { + "core" : { + "loki_chunks_bucket" : module.S3["loki"].bucket_name + "loki_ruler_bucket" : module.S3["loki"].bucket_name, + "loki_admin_bucket" : module.S3["loki"].bucket_name, + "loki_s3_region" : data.aws_region.current.name, + "loki_irsa_role_arn" : module.irsa["loki"].role_arn, + "velero_use_secret" : false, + "velero_irsa_role_arn" : module.irsa["velero"].role_arn, + "velero_bucket" : module.S3["velero"].bucket_name, + "velero_bucket_region" : data.aws_region.current.name, + "velero_bucket_provider_url" : "", + "velero_bucket_credential_name" : "", + "velero_bucket_credential_key" : "", + "grafana_ha" : true, + "grafana_pg_host" : element(split(":", module.db.db_instance_endpoint), 0), + "grafana_pg_port" : var.db_port, + "grafana_pg_database" : var.db_name, + "grafana_pg_password" : random_password.db_password.result, + "grafana_pg_user" : var.username + } + } + }) +} diff --git a/.github/test-infra/aws/rke2/irsa.tf b/.github/test-infra/aws/rke2/irsa.tf index 8ab35fea6..85cf380c1 100644 --- a/.github/test-infra/aws/rke2/irsa.tf +++ b/.github/test-infra/aws/rke2/irsa.tf @@ -50,7 +50,6 @@ module "oidc_bucket" { restrict_public_buckets = false } - # OIDC file creation resource "local_file" "oidc_config" { content = < /dev/null -export KUBECONFIG=$(pwd)/rke2-config +mkdir -p /home/runner/.kube +mv ./rke2-config /home/runner/.kube/config +#export KUBECONFIG=$(pwd)/rke2-config # find existing host record in the host file and save the line numbers matches_in_hosts="$(grep -n $cluster_hostname /etc/hosts | cut -f1 -d:)" diff --git a/.github/test-infra/aws/rke2/uds-config.tf b/.github/test-infra/aws/rke2/uds-config.tf new file mode 100644 index 000000000..1ea9dd225 --- /dev/null +++ b/.github/test-infra/aws/rke2/uds-config.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +resource "local_sensitive_file" "uds_config" { + filename = "../../../bundles/rke2/uds-config.yaml" + content = yamlencode({ + "options" : { + "architecture" : "amd64" + }, + "variables" : { + "core" : { + "loki_chunks_bucket" : module.storage.s3_buckets["loki"].bucket_name + "loki_ruler_bucket" : module.storage.s3_buckets["loki"].bucket_name, + "loki_admin_bucket" : module.storage.s3_buckets["loki"].bucket_name, + "loki_s3_region" : data.aws_region.current.name, + "loki_irsa_role_arn" : module.storage.irsa["loki"].bucket_role.arn + "velero_use_secret" : false, + "velero_irsa_role_arn" : module.storage.irsa["velero"].bucket_role.arn, + "velero_bucket" : module.storage.s3_buckets["velero"].bucket_name, + "velero_bucket_region" : data.aws_region.current.name, + "velero_bucket_provider_url" : "" + "velero_bucket_credential_name" : "", + "velero_bucket_credential_key" : "", + "grafana_ha" : false, + "grafana_pg_host" : "\"\"", + "grafana_pg_port" : "\"\"", + "grafana_pg_database" : "\"\"", + "grafana_pg_password" : "\"\"", + "grafana_pg_user" : "\"\"", + } + } + }) +} diff --git a/.github/test-infra/azure/aks/main.tf b/.github/test-infra/azure/aks/main.tf new file mode 100644 index 000000000..acc065e27 --- /dev/null +++ b/.github/test-infra/azure/aks/main.tf @@ -0,0 +1,108 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + + +data "azurerm_client_config" "current" {} + +locals { + cluster_name = "${var.cluster_name}-${random_string.name.result}" +} + +resource "random_string" "name" { + length = 4 + special = false + upper = false + numeric = false +} + +## resource group that cluster will be created in +resource "azurerm_resource_group" "this" { + name = "${var.resource_group_name}-${random_string.name.result}" + location = var.location +} + +resource "azurerm_role_assignment" "cluster_admin" { + scope = azurerm_kubernetes_cluster.aks_cluster.id + role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_role_assignment" "aks_network_role" { + principal_id = azurerm_kubernetes_cluster.aks_cluster.identity[0].principal_id + role_definition_name = "Network Contributor" + scope = azurerm_resource_group.this.id +} + +### CSI Driver identity. Required if workload_identity_enabled is true +resource "azurerm_user_assigned_identity" "workload_identity" { + count = var.workload_identity_enabled ? 1 : 0 + location = var.location + name = "${local.cluster_name}-workload-identity" + resource_group_name = azurerm_resource_group.this.name +} + +resource "azurerm_kubernetes_cluster" "aks_cluster" { + name = local.cluster_name + location = var.location + resource_group_name = azurerm_resource_group.this.name + kubernetes_version = var.kubernetes_version + dns_prefix = var.dns_prefix + sku_tier = var.sku_tier + workload_identity_enabled = var.workload_identity_enabled + oidc_issuer_enabled = var.oidc_issuer_enabled + + default_node_pool { + name = var.default_node_pool_name + auto_scaling_enabled = var.enable_autoscaling + vnet_subnet_id = azurerm_subnet.cluster_node_subnet.id + max_count = var.autoscaling_max_node_count + min_count = var.autoscaling_min_node_count + vm_size = var.default_node_pool_vm_size + zones = var.default_node_pool_availability_zones + node_labels = var.default_node_pool_node_labels + max_pods = var.default_node_pool_max_pods + node_count = var.default_node_pool_node_count + os_disk_type = var.default_node_pool_os_disk_type + tags = var.tags + } + + identity { + type = "SystemAssigned" + } + + azure_active_directory_role_based_access_control { + azure_rbac_enabled = true + admin_group_object_ids = [] + } + + dynamic "key_vault_secrets_provider" { + for_each = var.enable_key_vault_csi_driver ? { "enabled" = true } : {} + content { + secret_rotation_enabled = key_vault_secrets_provider.value + } + } + + storage_profile { + blob_driver_enabled = false + file_driver_enabled = true + } + + network_profile { + dns_service_ip = var.network_dns_service_ip + network_plugin = var.network_plugin + network_policy = var.network_policy + outbound_type = var.outbound_type + service_cidr = var.network_service_cidr + } + + lifecycle { + ignore_changes = [ + kubernetes_version, + tags + ] + } + + depends_on = [ + azurerm_resource_group.this + ] +} diff --git a/.github/test-infra/azure/aks/networking.tf b/.github/test-infra/azure/aks/networking.tf new file mode 100644 index 000000000..39c1a8ed2 --- /dev/null +++ b/.github/test-infra/azure/aks/networking.tf @@ -0,0 +1,51 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +resource "azurerm_virtual_network" "cluster-vnet" { + name = "${local.cluster_name}-vnet" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.this.location + resource_group_name = azurerm_resource_group.this.name + tags = var.tags +} + +resource "azurerm_subnet" "cluster_node_subnet" { + name = "${local.cluster_name}-system-node-subnet" + resource_group_name = azurerm_resource_group.this.name + virtual_network_name = azurerm_virtual_network.cluster-vnet.name + address_prefixes = ["10.0.0.0/20"] +} + +# https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-networking-private +resource "azurerm_subnet" "postgres_subnet" { + name = "${local.cluster_name}-postgres-subnet" + resource_group_name = azurerm_resource_group.this.name + virtual_network_name = azurerm_virtual_network.cluster-vnet.name + address_prefixes = ["10.0.16.0/20"] + service_endpoints = [ + "Microsoft.Storage", + ] + delegation { + name = "fs" + service_delegation { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + ] + } + } +} + + +resource "azurerm_private_dns_zone" "cluster_dns_zone" { + name = "${local.cluster_name}.postgres.database.azure.com" + resource_group_name = azurerm_resource_group.this.name +} + +resource "azurerm_private_dns_zone_virtual_network_link" "cluster_dns_zone_link" { + name = "${local.cluster_name}-dns" + private_dns_zone_name = azurerm_private_dns_zone.cluster_dns_zone.name + virtual_network_id = azurerm_virtual_network.cluster-vnet.id + resource_group_name = azurerm_resource_group.this.name + depends_on = [azurerm_subnet.postgres_subnet] +} \ No newline at end of file diff --git a/.github/test-infra/azure/aks/outputs.tf b/.github/test-infra/azure/aks/outputs.tf new file mode 100644 index 000000000..e24f96e82 --- /dev/null +++ b/.github/test-infra/azure/aks/outputs.tf @@ -0,0 +1,65 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +output "storage_account_name" { + description = "Specifies the name of the storage account" + value = azurerm_storage_account.cluster_storage.name + sensitive = true +} + +output "storage_account_access_key" { + description = "Specifies the primary access key of the storage account" + value = azurerm_storage_account.cluster_storage.primary_access_key + sensitive = true +} + +output "resource_group_name" { + value = azurerm_resource_group.this.name + sensitive = true +} + +output "loki_blob_container_name" { + value = azurerm_storage_container.loki_container.name + sensitive = true +} + +output "velero_blob_container_name" { + value = azurerm_storage_container.velero_container.name + sensitive = true +} + +output "subscription_id" { + value = data.azurerm_client_config.current.subscription_id + sensitive = true +} + +output "grafana_pg_host" { + description = "DB Endpoint for Grafana" + value = azurerm_postgresql_flexible_server.grafana_psql_server.fqdn + sensitive = true +} + +output "grafana_pg_port" { + description = "DB Port for Grafana" + value = var.db_port +} + +output "grafana_pg_database" { + description = "Database name for Grafana" + value = var.db_name +} + +output "grafana_pg_user" { + description = "Database username for Grafana" + value = var.username +} + +output "grafana_pg_password" { + description = "RDS Password for Grafana" + value = random_password.db_password.result + sensitive = true +} + +output "grafana_ha" { + value = true +} \ No newline at end of file diff --git a/.github/test-infra/azure/aks/storage.tf b/.github/test-infra/azure/aks/storage.tf new file mode 100644 index 000000000..388bf4973 --- /dev/null +++ b/.github/test-infra/azure/aks/storage.tf @@ -0,0 +1,61 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +resource "azurerm_storage_account" "cluster_storage" { + name = substr("sa${replace(local.cluster_name, "-", "")}", 0, 24) + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + allow_nested_items_to_be_public = false + account_tier = "Standard" + account_replication_type = "GRS" + + tags = var.tags +} + + +# Create the container for Velero +resource "azurerm_storage_container" "velero_container" { + name = "velero" + storage_account_name = azurerm_storage_account.cluster_storage.name + container_access_type = "private" +} + +# Create the container for loki +resource "azurerm_storage_container" "loki_container" { + name = "loki" + storage_account_name = azurerm_storage_account.cluster_storage.name + container_access_type = "private" +} + +resource "random_password" "db_password" { + length = 16 + special = false +} + +resource "azurerm_postgresql_flexible_server" "grafana_psql_server" { + name = "${local.cluster_name}-grafana-psqlserver" + resource_group_name = azurerm_resource_group.this.name + location = azurerm_resource_group.this.location + version = "16" + delegated_subnet_id = azurerm_subnet.postgres_subnet.id + public_network_access_enabled = false + administrator_login = var.username + administrator_password = random_password.db_password.result + + private_dns_zone_id = azurerm_private_dns_zone.cluster_dns_zone.id + storage_mb = 32768 + storage_tier = "P30" + + sku_name = "GP_Standard_D4s_v3" + lifecycle { + ignore_changes = [zone] + } + +} + +resource "azurerm_postgresql_flexible_server_database" "grafana_psql_db" { + name = var.db_name + server_id = azurerm_postgresql_flexible_server.grafana_psql_server.id + collation = "en_US.utf8" + charset = "utf8" +} diff --git a/.github/test-infra/azure/aks/uds-config.tf b/.github/test-infra/azure/aks/uds-config.tf new file mode 100644 index 000000000..4fb315a80 --- /dev/null +++ b/.github/test-infra/azure/aks/uds-config.tf @@ -0,0 +1,34 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +resource "local_sensitive_file" "uds_config" { + filename = "../../../bundles/aks/uds-config.yaml" + content = yamlencode({ + "options" : { + "architecture" : "amd64" + }, + "variables" : { + "core" : { + "azure_loki_storage_account" : azurerm_storage_account.cluster_storage.name, + "azure_loki_storage_account_access_key" : azurerm_storage_account.cluster_storage.primary_access_key, + "azure_loki_storage_account_container" : azurerm_storage_container.loki_container.name, + "azure_velero_storage_account" : azurerm_storage_account.cluster_storage.name, + "azure_velero_storage_account_access_key" : azurerm_storage_account.cluster_storage.primary_access_key + "azure_velero_storage_account_container" : azurerm_storage_container.velero_container.name, + "azure_subscription_id" : data.azurerm_client_config.current.subscription_id, + "azure_resource_group" : azurerm_resource_group.this.name, + "node_resource_group_name" : azurerm_kubernetes_cluster.aks_cluster.node_resource_group + "grafana_pg_host" : azurerm_postgresql_flexible_server.grafana_psql_server.fqdn, + "grafana_pg_port" : var.db_port, + "grafana_pg_database" : var.db_name, + "grafana_pg_password" : random_password.db_password.result, + "grafana_pg_user" : var.username, + } + } + }) +} + +resource "local_sensitive_file" "kubeconfig" { + filename = "/home/runner/.kube/config" + content = azurerm_kubernetes_cluster.aks_cluster.kube_admin_config_raw +} \ No newline at end of file diff --git a/.github/test-infra/azure/aks/variables.tf b/.github/test-infra/azure/aks/variables.tf new file mode 100644 index 000000000..0e1b18b7f --- /dev/null +++ b/.github/test-infra/azure/aks/variables.tf @@ -0,0 +1,194 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +variable "cluster_name" { + description = "(Required) Specifies the name of the AKS cluster." + type = string + default = "nightly-ci-aks" +} + +variable "resource_group_name" { + description = "(Required) Specifies the name of the resource group that will be created. Resources created in this terraform will be created under this resource group." + type = string +} + +variable "location" { + description = "(Required) Specifies the location where the AKS cluster will be deployed." + type = string +} + +variable "dns_prefix" { + description = "(Optional) DNS prefix specified when creating the managed cluster. Changing this forces a new resource to be created." + type = string + default = "nightly-ci" +} + +variable "sku_tier" { + description = "(Optional) The SKU Tier that should be used for this Kubernetes Cluster. Possible values are Free and Paid (which includes the Uptime SLA). Defaults to Free." + default = "Free" + type = string + + validation { + condition = contains(["Free", "Paid"], var.sku_tier) + error_message = "The sku tier is invalid." + } +} + +variable "kubernetes_version" { + description = "Specifies the AKS Kubernetes version" + default = "1.30" + type = string +} + +variable "enable_autoscaling" { + description = "(Optional) Enable cluster-autoscaler on all nodepools. Defaults to true." + type = bool + default = true +} + +variable "autoscaling_max_node_count" { + description = "The maximum number of nodes to allow the default (system) node pool to scale up to." + type = number + default = 6 +} + +variable "autoscaling_min_node_count" { + description = "The minimum number of nodes that should always be present in the default (system) node pool." + type = number + default = 4 +} + +variable "default_node_pool_vm_size" { + description = "Specifies the vm size of the default node pool" + default = "Standard_F8s_v2" + type = string +} + +variable "default_node_pool_availability_zones" { + description = "Specifies the availability zones of the default node pool" + default = ["1", "2", "3"] + type = list(string) +} + +variable "network_dns_service_ip" { + description = "Specifies the DNS service IP" + default = "10.2.0.10" + type = string +} + +variable "network_service_cidr" { + description = "Specifies the service CIDR" + default = "10.2.0.0/24" + type = string +} + +variable "network_plugin" { + description = "Specifies the network plugin of the AKS cluster" + default = "azure" + type = string +} + +variable "network_policy" { + description = "Specifies the network policy to use" + default = "azure" + type = string +} + +variable "outbound_type" { + description = "(Optional) The outbound (egress) routing method which should be used for this Kubernetes Cluster. Possible values are loadBalancer and userDefinedRouting. Defaults to loadBalancer." + type = string + default = "loadBalancer" + + validation { + condition = contains(["loadBalancer", "userDefinedRouting"], var.outbound_type) + error_message = "The outbound type is invalid." + } +} + +variable "default_node_pool_name" { + description = "Specifies the name of the default node pool" + default = "system" + type = string +} + +variable "default_node_pool_max_pods" { + description = "(Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + type = number + default = 50 +} + +variable "default_node_pool_node_labels" { + description = "(Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g key=value:NoSchedule). Changing this forces a new resource to be created." + type = map(any) + default = {} +} + +variable "default_node_pool_os_disk_type" { + description = "(Optional) The type of disk which should be used for the Operating System. Possible values are Ephemeral and Managed. Defaults to Managed. Changing this forces a new resource to be created." + type = string + default = "Ephemeral" +} + +variable "default_node_pool_node_count" { + description = "(Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between 0 and 1000 and must be a value in the range min_count - max_count." + type = number + default = 4 +} + +variable "tags" { + description = "(Optional) Specifies the tags of the bastion host" + type = map(any) + default = {} +} + +variable "azure_rbac_enabled" { + description = "Whether or not to use Azure Role Based Access Control to control access to cluster resources." + default = true +} + +variable "enable_key_vault_csi_driver" { + description = "(Optional) Whether or not to deploy the Azure Key Vault CSI driver managed add-on. Defaults to false." + type = bool + default = false +} + +variable "workload_identity_enabled" { + description = "(Optional) Specifies whether Microsoft Entra ID Workload Identity should be enabled for the Cluster. Defaults to false." + type = bool + default = true +} + +variable "cluster_managed_identity_type" { + description = "Type of Managed Identity to be used for the cluster. Valid types are SystemAssigned or UserAssigned." + type = string + default = "SystemAssigned" +} + +variable "cluster_kubelet_identity_type" { + description = "Type of Managed Identity to be used for Kubelet. If UserAssigned, defaults to Azure automically creating Managaged Identity for Kubelet." + default = "SystemAssigned" +} + +variable "oidc_issuer_enabled" { + description = "(Optional) Enable or Disable the OIDC issuer URL." + type = bool + default = true +} + +variable "username" { + description = "The username to use to login to the DB" + type = string + default = "grafana" +} + +variable "db_port" { + description = "The database port" + type = number + default = 5432 +} + +variable "db_name" { + description = "The name to give the database" + type = string + default = "grafana" +} \ No newline at end of file diff --git a/.github/test-infra/azure/aks/versions.tf b/.github/test-infra/azure/aks/versions.tf new file mode 100644 index 000000000..7dabd826c --- /dev/null +++ b/.github/test-infra/azure/aks/versions.tf @@ -0,0 +1,12 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +terraform { + backend "azurerm" { + } +} + +provider "azurerm" { + features { + } +} diff --git a/.github/workflows/nightly-testing.yaml b/.github/workflows/nightly-testing.yaml deleted file mode 100644 index cb7fb10c0..000000000 --- a/.github/workflows/nightly-testing.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2024 Defense Unicorns -# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial - -name: Nightly Testing - -on: - schedule: - - cron: '0 6 * * *' # Runs at midnight Mountain every day - pull_request: - paths: - - tasks/iac.yaml - - .github/workflows/test-eks.yaml - - .github/workflows/test-rke2.yaml - - .github/bundles/eks/* - - .github/bundles/rke2/* - - .github/test-infra/aws/eks/* - - .github/test-infra/aws/rke2/* - -# Abort prior jobs in the same workflow / PR -concurrency: - group: test-eks-dev-${{ github.ref }} - cancel-in-progress: true - -jobs: - nightly-testing-eks: - name: Test Core on EKS - uses: ./.github/workflows/test-eks.yaml - strategy: - matrix: - flavor: [upstream, registry1, unicorn] - fail-fast: false - with: - flavor: ${{ matrix.flavor }} - secrets: inherit - nightly-testing-rke2: - name: Test Core on RKE2 - uses: ./.github/workflows/test-rke2.yaml - strategy: - matrix: - flavor: [upstream, registry1, unicorn] - fail-fast: false - with: - flavor: ${{ matrix.flavor }} - secrets: inherit diff --git a/.github/workflows/test-aks.yaml b/.github/workflows/test-aks.yaml new file mode 100644 index 000000000..ad77cd014 --- /dev/null +++ b/.github/workflows/test-aks.yaml @@ -0,0 +1,97 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +name: AKS Nightly Test + +on: + schedule: + - cron: '0 6 * * *' # Runs at midnight Mountain every day + pull_request: + paths: + - tasks/iac.yaml + - .github/bundles/aks/* + - .github/test-infra/azure/aks/* + +permissions: + id-token: write + contents: read + +jobs: + test-aks-install: + strategy: + matrix: + flavor: [upstream, registry1, unicorn] + runs-on: ubuntu-latest + env: + SHA: ${{ github.sha }} + UDS_REGION: centralus + UDS_RESOURCE_GROUP_NAME: ${{ secrets.AZURE_RESOURCE_GROUP }} + UDS_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} + UDS_CONTAINER_NAME: ${{ secrets.AZURE_STORAGE_CONTAINER_NAME }} + + steps: + - name: Set ENV + run: | + echo "UDS_CLUSTER_NAME=uds-ci-${{ matrix.flavor }}-${SHA:0:7}" >> $GITHUB_ENV + echo "UDS_STATE_KEY="uds-core/${SHA:0:7}-nightly-aks-core-${{ matrix.flavor }}.tfstate >> $GITHUB_ENV + echo "TF_VAR_location=${UDS_REGION}" >> $GITHUB_ENV + echo "TF_VAR_cluster_name=uds-ci-${{ matrix.flavor }}-${SHA:0:7}" >> $GITHUB_ENV + echo "TF_VAR_resource_group_name=uds-ci-${{ matrix.flavor }}" >> $GITHUB_ENV + echo "ARM_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV + echo "ARM_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV + echo "ARM_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV + echo "ARM_USE_OIDC=true" >> $GITHUB_ENV + echo "ARM_STORAGE_USE_AZUREAD=true" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Environment setup + uses: ./.github/actions/setup + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: Setup Tofu + uses: opentofu/setup-opentofu@12f4debbf681675350b6cd1f0ff8ecfbda62027b # v1.0.4 + with: + tofu_version: 1.8.2 + tofu_wrapper: false + + - name: Create UDS Core Package + run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} + + - name: Create Core Bundle + run: uds create .github/bundles/aks --confirm + + - name: Create IAC + run: uds run -f tasks/iac.yaml apply-tofu --no-progress --set K8S_DISTRO=aks --set CLOUD=azure + + - name: Deploy Core Bundle + env: + UDS_CONFIG: .github/bundles/aks/uds-config.yaml + run: uds deploy .github/bundles/aks/uds-bundle-uds-core-aks-nightly-*.tar.zst --confirm + timeout-minutes: 30 + + - name: Debug Output + if: ${{ always() }} + uses: ./.github/actions/debug-output + + - name: Remove UDS Core + if: always() + run: uds remove .github/bundles/aks/uds-bundle-uds-core-aks-nightly-*.tar.zst --confirm + timeout-minutes: 10 + continue-on-error: true + + - name: Remove IAC + if: always() + run: uds run -f tasks/iac.yaml destroy-iac --no-progress --set K8S_DISTRO=aks --set CLOUD=azure diff --git a/.github/workflows/test-eks.yaml b/.github/workflows/test-eks.yaml index 727e35022..4587b6537 100644 --- a/.github/workflows/test-eks.yaml +++ b/.github/workflows/test-eks.yaml @@ -1,15 +1,16 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial -name: Test Core On EKS +name: EKS Nightly Test on: - # triggered by nightly-testing.yaml - workflow_call: - inputs: - flavor: - required: true - type: string + schedule: + - cron: '0 6 * * *' # Runs at midnight Mountain every day + pull_request: + paths: + - tasks/iac.yaml + - .github/bundles/eks/* + - .github/test-infra/aws/eks/* permissions: id-token: write @@ -17,6 +18,9 @@ permissions: jobs: test-eks-install: + strategy: + matrix: + flavor: [upstream, registry1, unicorn] runs-on: ubuntu-latest env: SHA: ${{ github.sha }} @@ -28,10 +32,10 @@ jobs: steps: - name: Set ENV run: | - echo "UDS_CLUSTER_NAME=uds-ci-${{ inputs.flavor }}-${SHA:0:7}" >> $GITHUB_ENV - echo "UDS_STATE_KEY="tfstate/ci/install/${SHA:0:7}-eks-core-${{ inputs.flavor }}-aws.tfstate >> $GITHUB_ENV + echo "UDS_CLUSTER_NAME=uds-ci-${{ matrix.flavor }}-${SHA:0:7}" >> $GITHUB_ENV + echo "UDS_STATE_KEY="tfstate/ci/install/${SHA:0:7}-eks-core-${{ matrix.flavor }}-aws.tfstate >> $GITHUB_ENV echo "TF_VAR_region=${UDS_REGION}" >> $GITHUB_ENV - echo "TF_VAR_name=uds-ci-${{ inputs.flavor }}-${SHA:0:7}" >> $GITHUB_ENV + echo "TF_VAR_name=uds-ci-${{ matrix.flavor }}-${SHA:0:7}" >> $GITHUB_ENV echo "TF_VAR_use_permissions_boundary=true" >> $GITHUB_ENV echo "TF_VAR_permissions_boundary_name=${UDS_PERMISSIONS_BOUNDARY_NAME}" >> $GITHUB_ENV @@ -64,7 +68,7 @@ jobs: tofu_wrapper: false - name: Create UDS Core Package - run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ inputs.flavor }} + run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - name: Create Core Bundle run: uds create .github/bundles/eks --confirm @@ -74,7 +78,7 @@ jobs: timeout-minutes: 60 - name: Create IAC - run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=eks + run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=eks --set CLOUD=aws timeout-minutes: 20 - name: Deploy Core Bundle @@ -83,6 +87,10 @@ jobs: run: uds deploy .github/bundles/eks/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm timeout-minutes: 20 + - name: Debug Output + if: ${{ always() }} + uses: ./.github/actions/debug-output + - name: Remove UDS Core if: always() run: uds remove .github/bundles/eks/uds-bundle-uds-core-eks-*.tar.zst --confirm @@ -91,7 +99,7 @@ jobs: - name: Remove IAC if: always() - run: uds run -f tasks/iac.yaml destroy-iac --no-progress --set K8S_DISTRO=eks + run: uds run -f tasks/iac.yaml destroy-iac --no-progress --set K8S_DISTRO=eks --set CLOUD=aws timeout-minutes: 10 continue-on-error: true diff --git a/.github/workflows/test-rke2.yaml b/.github/workflows/test-rke2.yaml index 8bd6480b2..c21e0ceba 100644 --- a/.github/workflows/test-rke2.yaml +++ b/.github/workflows/test-rke2.yaml @@ -1,15 +1,16 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial -name: Test Core On RKE2 +name: RKE2 Nightly Test on: - # triggered by nightly-testing.yaml - workflow_call: - inputs: - flavor: - required: true - type: string + schedule: + - cron: '0 6 * * *' # Runs at midnight Mountain every day + pull_request: + paths: + - tasks/iac.yaml + - .github/bundles/rke2/* + - .github/test-infra/aws/rke2/* permissions: id-token: write @@ -17,6 +18,9 @@ permissions: jobs: test-rke2-install: + strategy: + matrix: + flavor: [upstream, registry1, unicorn] runs-on: ubuntu-latest env: SHA: ${{ github.sha }} @@ -28,8 +32,8 @@ jobs: steps: - name: Set ENV run: | - echo "UDS_CLUSTER_NAME=uds-ci-${{ inputs.flavor }}-${SHA:0:7}" >> $GITHUB_ENV - echo "UDS_STATE_KEY="tfstate/ci/install/${SHA:0:7}-rke2-core-${{ inputs.flavor }}-aws.tfstate >> $GITHUB_ENV + echo "UDS_CLUSTER_NAME=uds-ci-${{ matrix.flavor }}-${SHA:0:7}" >> $GITHUB_ENV + echo "UDS_STATE_KEY="tfstate/ci/install/${SHA:0:7}-rke2-core-${{ matrix.flavor }}-aws.tfstate >> $GITHUB_ENV echo "TF_VAR_region=${UDS_REGION}" >> $GITHUB_ENV echo "TF_VAR_run_id=$GITHUB_RUN_ID" >> $GITHUB_ENV echo "TF_VAR_use_permissions_boundary=true" >> $GITHUB_ENV @@ -64,13 +68,13 @@ jobs: tofu_wrapper: false - name: Create UDS Core Package - run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ inputs.flavor }} + run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - name: Create Core Bundle run: uds create .github/bundles/rke2 --confirm - name: Create IAC - run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=rke2 + run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=rke2 --set CLOUD=aws timeout-minutes: 20 - name: Get Kubeconfig @@ -79,17 +83,18 @@ jobs: - name: Wait for RKE2 Cluster run: uds run -f tasks/iac.yaml rke2-cluster-ready --no-progress - env: - KUBECONFIG: ".github/test-infra/aws/rke2/rke2-config" timeout-minutes: 20 - name: Deploy Core Bundle env: - KUBECONFIG: ".github/test-infra/aws/rke2/rke2-config" UDS_CONFIG: .github/bundles/rke2/uds-config.yaml run: uds deploy .github/bundles/rke2/uds-bundle-uds-core-rke2-nightly-*.tar.zst --confirm timeout-minutes: 60 + - name: Debug Output + if: ${{ always() }} + uses: ./.github/actions/debug-output + - name: Remove UDS Core if: always() run: uds remove .github/bundles/rke2/uds-bundle-uds-core-rke2-*.tar.zst --confirm @@ -100,6 +105,6 @@ jobs: - name: Remove IAC if: always() - run: uds run -f tasks/iac.yaml destroy-iac --no-progress --set K8S_DISTRO=rke2 + run: uds run -f tasks/iac.yaml destroy-iac --no-progress --set K8S_DISTRO=rke2 --set CLOUD=aws timeout-minutes: 10 continue-on-error: true diff --git a/tasks/iac.yaml b/tasks/iac.yaml index 0c178bab9..9acbbc8e5 100644 --- a/tasks/iac.yaml +++ b/tasks/iac.yaml @@ -5,12 +5,16 @@ variables: - name: CLUSTER_NAME - name: K8S_DISTRO + - name: CLOUD - name: REGION - name: PERMISSIONS_BOUNDARY_NAME - name: PERMISSIONS_BOUNDARY_ARN - name: STATE_BUCKET_NAME - name: STATE_DYNAMODB_TABLE_NAME - name: STATE_KEY + - name: RESOURCE_GROUP_NAME + - name: STORAGE_ACCOUNT_NAME + - name: CONTAINER_NAME - name: AMI_ID default: ami-068ab6ac1cec494e0 @@ -99,7 +103,6 @@ tasks: actions: - task: rke2-nodes-ready - cmd: | - export KUBECONFIG=./rke2-config # wait for at least 3 nodes while true; do if [ $(uds zarf tools kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[-1].type}={.status.conditions[-1].status}{"\n"}{end}' | egrep -i '^ready.*true' | wc -l) -lt 3 ]; then @@ -131,115 +134,30 @@ tasks: - name: create-iac actions: - task: apply-tofu - - task: tofu-outputs - - task: create-uds-config - name: destroy-iac actions: - cmd: tofu destroy -auto-approve - dir: .github/test-infra/aws/${K8S_DISTRO} + dir: .github/test-infra/${CLOUD}/${K8S_DISTRO} - name: apply-tofu actions: - - cmd: echo ${STATE_KEY} | sed 's/\.tfstate/-buckets.tfstate/g' - setVariables: - - name: BUCKETS_STATE_KEY - dir: .github/test-infra/aws/${K8S_DISTRO} - cmd: | - tofu init -force-copy \ - -backend-config="bucket=${STATE_BUCKET_NAME}" \ - -backend-config="key=${BUCKETS_STATE_KEY}" \ - -backend-config="region=${REGION}" \ - -backend-config="dynamodb_table=${STATE_DYNAMODB_TABLE_NAME}" - dir: .github/test-infra/aws/${K8S_DISTRO} + if [ ${CLOUD} = "aws" ]; then + tofu init -force-copy \ + -backend-config="bucket=${STATE_BUCKET_NAME}" \ + -backend-config="key=${STATE_KEY}" \ + -backend-config="region=${REGION}" \ + -backend-config="dynamodb_table=${STATE_DYNAMODB_TABLE_NAME}" + elif [ ${CLOUD} = "azure" ]; then + tofu init -force-copy \ + -backend=true \ + -backend-config="resource_group_name=$RESOURCE_GROUP_NAME" \ + -backend-config="storage_account_name=$STORAGE_ACCOUNT_NAME" \ + -backend-config="container_name=$CONTAINER_NAME" \ + -backend-config="key=${STATE_KEY}" + else + echo "Invalid cloud provider specified."; return 1; fi + dir: .github/test-infra/${CLOUD}/${K8S_DISTRO} - cmd: tofu apply -auto-approve - dir: .github/test-infra/aws/${K8S_DISTRO} - - - name: tofu-outputs - actions: - - cmd: tofu output -raw loki_s3_bucket - setVariables: - - name: "LOKI_S3_BUCKET" - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw aws_region - setVariables: - - name: LOKI_S3_AWS_REGION - dir: .github/test-infra/aws/${K8S_DISTRO} - - cmd: tofu output -raw loki_irsa_role_arn - setVariables: - - name: LOKI_S3_ROLE_ARN - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw velero_s3_bucket - setVariables: - - name: VELERO_S3_BUCKET - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw aws_region - setVariables: - - name: VELERO_S3_AWS_REGION - dir: .github/test-infra/aws/${K8S_DISTRO} - - cmd: tofu output -raw velero_irsa_role_arn - setVariables: - - name: VELERO_S3_ROLE_ARN - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw grafana_pg_host 2>/dev/null || echo '""' - setVariables: - - name: GRAFANA_PG_HOST - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw grafana_pg_port 2>/dev/null || echo '""' - setVariables: - - name: GRAFANA_PG_PORT - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw grafana_pg_database 2>/dev/null || echo '""' - setVariables: - - name: GRAFANA_PG_DATABASE - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw grafana_pg_user 2>/dev/null || echo '""' - setVariables: - - name: GRAFANA_PG_USER - dir: .github/test-infra/aws/${K8S_DISTRO} - mute: true - - cmd: tofu output -raw grafana_pg_password 2>/dev/null || echo '""' - setVariables: - - name: GRAFANA_PG_PASSWORD - mute: true # Muted to hide sensitive password - dir: .github/test-infra/aws/${K8S_DISTRO} - - cmd: tofu output -raw grafana_ha - setVariables: - - name: GRAFANA_HA - dir: .github/test-infra/aws/${K8S_DISTRO} - - - name: create-uds-config - actions: - - task: tofu-outputs - - cmd: | - cat < .github/bundles/${K8S_DISTRO}/uds-config.yaml - options: - architecture: amd64 - variables: - core: - loki_chunks_bucket: ${LOKI_S3_BUCKET} - loki_ruler_bucket: ${LOKI_S3_BUCKET} - loki_admin_bucket: ${LOKI_S3_BUCKET} - loki_s3_region: ${LOKI_S3_AWS_REGION} - loki_irsa_role_arn: ${LOKI_S3_ROLE_ARN} - velero_use_secret: false - velero_irsa_role_arn: ${VELERO_S3_ROLE_ARN} - velero_bucket: ${VELERO_S3_BUCKET} - velero_bucket_region: ${VELERO_S3_AWS_REGION} - velero_bucket_provider_url: "" - velero_bucket_credential_name: "" - velero_bucket_credential_key: "" - grafana_ha: ${GRAFANA_HA} - grafana_pg_host: ${GRAFANA_PG_HOST} - grafana_pg_port: ${GRAFANA_PG_PORT} - grafana_pg_database: ${GRAFANA_PG_DATABASE} - grafana_pg_password: ${GRAFANA_PG_PASSWORD} - grafana_pg_user: ${GRAFANA_PG_USER} - EOF + dir: .github/test-infra/${CLOUD}/${K8S_DISTRO}