Prepare Flux for multi-tenancy support (#1608)

* Add missing dependency between Grafana helm chart and the resources it uses from the kube-prometheus-stack * Prepare for multi-tenancy support * Add support for Flux multi-tenancy * Remove unused variables. Add support for Trivy in pre-commit. * Prepare druid-operator for Flux multi-tenancy * Prepare Trivy Operator for Flux multi-tenancy
dfds · Nov 8, 2024 · aed01a7 · aed01a7
1 parent 66cdedc
commit aed01a7
Show file tree

Hide file tree

Showing 46 changed files with 402 additions and 121 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,35 +1,37 @@
 repos:
-- repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.96.2
-  hooks:
-    - id: terraform_fmt
-      exclude: test/integration/suite/vendor/.*
-    - id: terraform_tfsec
-      exclude: test/integration/suite/vendor/.*
-      files: (\.tf|\.tfvars)$
-- repo: https://github.com/Yelp/detect-secrets
-  rev: v1.5.0
-  hooks:
-    - id: detect-secrets
-      exclude: test/integration/suite/vendor/.*
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
-  hooks:
-    - id: trailing-whitespace
-      exclude: test/integration/suite/vendor/.*
-    - id: check-added-large-files
-      exclude: test/integration/suite/vendor/.*
-    - id: check-merge-conflict
-    - id: detect-aws-credentials
-      exclude: test/integration/suite/vendor/.*
-      args:
-        - "--allow-missing-credentials"
-    - id: detect-private-key
-- repo: https://github.com/gruntwork-io/pre-commit
-  rev: v0.1.24
-  hooks:
-    - id: tflint
-      exclude: test/integration/suite/vendor/.*
-      args:
-        - "--module"
-        - "--config=.tflint.hcl"
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.96.2
+    hooks:
+      - id: terraform_fmt
+        exclude: test/integration/suite/vendor/.*
+      - id: terraform_trivy
+        args:
+          - "--args=--skip-dirs=test/integration/suite/vendor/.*"
+          - "--args=--severity HIGH,CRITICAL"
+        files: (\.tf|\.tfvars)$
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
+    hooks:
+      - id: detect-secrets
+        exclude: test/integration/suite/vendor/.*
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: test/integration/suite/vendor/.*
+      - id: check-added-large-files
+        exclude: test/integration/suite/vendor/.*
+      - id: check-merge-conflict
+      - id: detect-aws-credentials
+        exclude: test/integration/suite/vendor/.*
+        args:
+          - "--allow-missing-credentials"
+      - id: detect-private-key
+  - repo: https://github.com/gruntwork-io/pre-commit
+    rev: v0.1.24
+    hooks:
+      - id: tflint
+        exclude: test/integration/suite/vendor/.*
+        args:
+          - "--call-module-type=all"
+          - "--config=__GIT_ROOT__/.tflint.hcl"
diff --git a/_sub/compute/atlantis/flux.tf b/_sub/compute/atlantis/flux.tf
@@ -7,6 +7,8 @@ resource "github_repository_file" "helm" {
   branch     = local.repo_branch
   file       = "${local.cluster_repo_path}/${local.app_install_name}-helm.yaml"
   content = templatefile("${path.module}/values/app-config.yaml", {
+    deploy_name      = local.deploy_name
+    namespace        = local.namespace
     app_install_name = local.app_install_name
     helm_repo_path   = local.helm_repo_path
     prune            = var.prune

diff --git a/_sub/compute/atlantis/values/app-config.yaml b/_sub/compute/atlantis/values/app-config.yaml
@@ -1,9 +1,24 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-reconciler-${deploy_name}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: helm-controller
+    namespace: ${namespace}
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: ${app_install_name}-helm
   namespace: flux-system
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   dependsOn:
     - name: platform-apps-sources

diff --git a/_sub/compute/atlantis/values/patch.yaml b/_sub/compute/atlantis/values/patch.yaml
@@ -4,6 +4,7 @@ metadata:
   name: ${deploy_name}
   namespace: ${namespace}
 spec:
+  serviceAccountName: helm-controller
   chart:
     spec:
       version: "${chart_version}"

diff --git a/_sub/compute/druid-operator/main.tf b/_sub/compute/druid-operator/main.tf
@@ -8,6 +8,8 @@ resource "github_repository_file" "helm" {
   content = templatefile("${path.module}/values/app-config.yaml", {
     app_install_name = local.app_install_name
     helm_repo_path   = local.helm_repo_path
+    deploy_name      = var.deploy_name
+    namespace        = var.namespace
     prune            = var.prune
   })
   overwrite_on_create = var.overwrite_on_create

diff --git a/_sub/compute/druid-operator/values/app-config.yaml b/_sub/compute/druid-operator/values/app-config.yaml
@@ -1,9 +1,24 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-reconciler-${deploy_name}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: helm-controller
+    namespace: ${namespace}
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: ${app_install_name}-helm
   namespace: flux-system
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   dependsOn:
     - name: platform-apps-sources

diff --git a/_sub/compute/druid-operator/values/patch.yaml b/_sub/compute/druid-operator/values/patch.yaml
@@ -4,6 +4,7 @@ metadata:
   name: ${deploy_name}
   namespace: ${namespace}
 spec:
+  serviceAccountName: helm-controller
   chart:
     spec:
       version: "${chart_version}"

diff --git a/_sub/compute/github-arc-runners/dependencies.tf b/_sub/compute/github-arc-runners/dependencies.tf
@@ -12,17 +12,31 @@ locals {
 
 locals {
   app_helm_path = <<YAML
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-reconciler-${var.deploy_name}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: helm-controller
+    namespace: ${var.namespace}
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: "${local.app_install_name}-helm"
   namespace: "flux-system"
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   dependsOn:
     - name: "platform-apps-sources"
     - name: ${var.controller_deploy_name}
-
   sourceRef:
     kind: GitRepository
     name: "flux-system"
@@ -49,6 +63,7 @@ metadata:
   name: ${var.deploy_name}
   namespace: ${var.namespace}
 spec:
+  serviceAccountName: helm-controller
   chart:
     spec:
       version: ${var.helm_chart_version}

diff --git a/_sub/compute/github-arc-runners/vars.tf b/_sub/compute/github-arc-runners/vars.tf
@@ -61,30 +61,30 @@ variable "prune" {
 }
 
 variable "github_config_url" {
-  type = string
+  type        = string
   description = "URL of Github organisation or repo for the runners"
 }
 
 variable "github_config_secret" {
-  type = string
+  type        = string
   description = "Secret name containing authorisation information for the runners. This is not deployed by this module, consider using external-secrets to deploy it"
 }
 
 variable "runner_scale_set_name" {
-  type = string
+  type        = string
   description = "Name for the runner scale set"
 }
 
 variable "storage_class_name" {
-  type = string
+  type        = string
   description = "Name of the storage class to use for the runners persistent volume"
-  default = "csi-gp3"
+  default     = "csi-gp3"
 }
 
 variable "storage_request_size" {
-  type = string
+  type        = string
   description = "Size of the persistent volume claim for the runners"
-  default = "1Gi"
+  default     = "1Gi"
 }
 
 variable "controller_deploy_name" {
@@ -94,25 +94,25 @@ variable "controller_deploy_name" {
 }
 
 variable "min_runners" {
-  type = number
+  type        = number
   description = "Minimum number of runners to keep running"
-  default = 0
+  default     = 0
 }
 
 variable "max_runners" {
-  type = number
+  type        = number
   description = "Maximum number of runners to keep running"
-  default = 5
+  default     = 5
 }
-  
+
 variable "runner_memory_request" {
-  type = string
+  type        = string
   description = "Memory request for the runner pods"
-  default = "128Mi"
+  default     = "128Mi"
 }
 
 variable "runner_memory_limit" {
-  type = string
+  type        = string
   description = "Memory request for the runner pods"
-  default = "8Gi"
+  default     = "8Gi"
 }
diff --git a/_sub/compute/github-arc-ss-controller/dependencies.tf b/_sub/compute/github-arc-ss-controller/dependencies.tf
@@ -12,16 +12,30 @@ locals {
 
 locals {
   app_helm_path = <<YAML
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-reconciler-${var.deploy_name}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: helm-controller
+    namespace: ${var.namespace}
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: "${local.app_install_name}-helm"
   namespace: "flux-system"
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   dependsOn:
     - name: "platform-apps-sources"
-
   sourceRef:
     kind: GitRepository
     name: "flux-system"
@@ -45,9 +59,10 @@ metadata:
   name: ${var.deploy_name}
   namespace: ${var.namespace}
 spec:
+  serviceAccountName: helm-controller
   chart:
     spec:
       version: ${var.helm_chart_version}
 YAML
 
-}
+}
diff --git a/_sub/compute/helm-kube-prometheus-stack/dependencies.tf b/_sub/compute/helm-kube-prometheus-stack/dependencies.tf
@@ -21,7 +21,8 @@ locals {
       "namespace" = "flux-system"
     }
     "spec" = {
-      "interval" = "1m0s"
+      "serviceAccountName" = "kustomize-controller"
+      "interval"           = "1m0s"
       "sourceRef" = {
         "kind" = "GitRepository"
         "name" = "flux-system"

diff --git a/_sub/compute/k8s-fluxcd/dependencies.tf b/_sub/compute/k8s-fluxcd/dependencies.tf
@@ -21,6 +21,7 @@ locals {
       "namespace" = "flux-system"
     }
     "spec" = {
+      "serviceAccountName" = "kustomize-controller"
       "dependsOn" = [
         {
           "name" = "platform-apps-sources"
@@ -62,6 +63,7 @@ metadata:
   name: platform-apps-sources
   namespace: flux-system
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   dependsOn:
     - name: flux-system
@@ -80,6 +82,7 @@ metadata:
   name: custom
   namespace: flux-system
 spec:
+  serviceAccountName: kustomize-controller
   interval: 1m0s
   path: ./platform-apps/${var.cluster_name}/custom
   prune: ${var.prune}
@@ -93,4 +96,3 @@ Place custom manifests in here. Make sure to place them in a folder named after
   EOT
 }
 
-
diff --git a/_sub/compute/k8s-fluxcd/main.tf b/_sub/compute/k8s-fluxcd/main.tf
@@ -24,9 +24,10 @@ resource "github_repository_deploy_key" "main" {
 }
 
 resource "flux_bootstrap_git" "this" {
-  depends_on = [github_repository_deploy_key.main]
-  path       = local.cluster_target_path
-  version    = var.release_tag
+  depends_on             = [github_repository_deploy_key.main]
+  path                   = local.cluster_target_path
+  version                = var.release_tag
+  kustomization_override = file("${path.module}/values/flux-system-patch.yaml")
 }