Add a helm golden record (#2034)

* Helm: Add golden-record build script * Helm: add test-values golden record * Add PR check for `check-helm-tests` * Add Helm setup to lint-helm action * Update generated helm tests * Fix bash linting * Update contribution guidelines * Update generated helm manifests * Helm: fix kube version Set kubeVersionOverride to generate PodDisruptionBudget API version consistently. When I ran the test, I got a diff, because my k8s is newer (1.23). Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com> * Update operations/helm/tests/build.sh Co-authored-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com> Co-authored-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>
grafana · Jun 7, 2022 · 84e4901 · 84e4901
1 parent 8415eb1
commit 84e4901
Show file tree

Hide file tree

Showing 101 changed files with 4,627 additions and 0 deletions.
diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml
@@ -71,6 +71,26 @@ jobs:
       - name: Check Jsonnet Tests
         run: make BUILD_IN_CONTAINER=false check-jsonnet-tests
 
+  lint-helm:
+    runs-on: ubuntu-latest
+    container:
+      image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+        # Commands in the Makefile are hardcoded with an assumed file structure of the CI container
+        # Symlink ensures paths specified in previous commands don’t break
+      - name: Symlink Expected Path to Workspace
+        run: |
+          mkdir -p /go/src/github.com/grafana/mimir
+          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.5.2
+      - name: Check Helm Tests
+        run: make BUILD_IN_CONTAINER=false check-helm-tests
+
   test:
     runs-on: ubuntu-latest
     strategy:

diff --git a/Makefile b/Makefile
@@ -496,6 +496,12 @@ check-jsonnet-getting-started:
 		| sed 's/\(jb install github.com\/grafana\/mimir\/operations\/mimir@main\)/\1 \&\& rm -fr .\/vendor\/mimir \&\& cp -r ..\/operations\/mimir .\/vendor\/mimir\//g' \
 		| bash
 
+build-helm-tests:
+	@./operations/helm/tests/build.sh
+
+check-helm-tests: build-helm-tests
+	@git diff --exit-code -- ./operations/helm/tests || (echo "Please rebuild helm tests output 'make build-helm-tests'" && false)
+
 build-jsonnet-tests:
 	@./operations/mimir-tests/build.sh
 

diff --git a/docs/internal/contributing/contributing-to-helm-chart.md b/docs/internal/contributing/contributing-to-helm-chart.md
@@ -6,6 +6,13 @@ Please see the [general workflow](README.md#workflow) for reference.
 
 - Changelog is in the chart itself [operations/helm/charts/mimir-distributed/CHANGELOG.md](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/CHANGELOG.md).
 - If you made any changes to the [operations/helm/charts/mimir-distributed/Chart.yaml](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/Chart.yaml), run `make doc` and commit the changed files to update the [operations/helm/charts/mimir-distributed/README.md](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/README.md).
+- If your changes impact the test configurations in the [operations/helm/charts/mimir-distributed/ci](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/ci) directory, see [Updating compiled manifests](#updating-compiled-manifests).
+
+## Updating compiled manifests
+
+We keep a compiled version of the helm chart for each values file in the `ci` directory.
+This makes it easy to see how a given PR impacts the final output.
+A PR check will fail if you forget to update the compiled manifests, and you can use `make build-helm-tests` to update them.
 
 ## Versioning
 

diff --git a/operations/helm/charts/mimir-distributed/ci/test-enterprise-values.yaml b/operations/helm/charts/mimir-distributed/ci/test-enterprise-values.yaml
@@ -1,4 +1,6 @@
 # Test values to limit the load during CI
+kubeVersionOverride: "1.20"
+
 enterprise:
   enabled: true
 

diff --git a/operations/helm/charts/mimir-distributed/ci/test-oss-values.yaml b/operations/helm/charts/mimir-distributed/ci/test-oss-values.yaml
@@ -1,4 +1,6 @@
 # Test values to limit the load during CI
+kubeVersionOverride: "1.20"
+
 alertmanager:
   persistentVolume:
     enabled: false

diff --git a/operations/helm/tests/build.sh b/operations/helm/tests/build.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+CHART_PATH=operations/helm/charts/mimir-distributed
+
+# Start from a clean slate
+rm -rf operations/helm/tests/*-generated
+
+# Install chart dependencies for this branch
+helm dependency update "$CHART_PATH"
+
+# Locally render the chart for every test file
+TESTS=$(ls -1 ${CHART_PATH}/ci/*values.yaml)
+
+for FILEPATH in $TESTS; do
+  # Extract the filename (without extension).
+  TEST_NAME=$(basename -s '.yaml' "$FILEPATH")
+
+  echo "Templating $TEST_NAME"
+  helm template "${TEST_NAME}" ${CHART_PATH} -f "${FILEPATH}" --output-dir "operations/helm/tests/${TEST_NAME}-generated"
+done
diff --git a/.../test-enterprise-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml b/.../test-enterprise-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml
@@ -0,0 +1,111 @@
+---
+# Source: mimir-distributed/charts/minio/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-enterprise-values-minio
+  labels:
+    app: minio
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm
+data:
+  initialize: |-
+    #!/bin/sh
+    set -e ; # Have script exit in the event of a failed command.
+    MC_CONFIG_DIR="/etc/minio/mc/"
+    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
+    
+    # connectToMinio
+    # Use a check-sleep-check loop to wait for Minio service to be available
+    connectToMinio() {
+      SCHEME=$1
+      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
+      set -e ; # fail if we can't read the keys.
+      ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ;
+      set +e ; # The connections to minio are allowed to fail.
+      echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
+      MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
+      $MC_COMMAND ;
+      STATUS=$? ;
+      until [ $STATUS = 0 ]
+      do
+        ATTEMPTS=`expr $ATTEMPTS + 1` ;
+        echo \"Failed attempts: $ATTEMPTS\" ;
+        if [ $ATTEMPTS -gt $LIMIT ]; then
+          exit 1 ;
+        fi ;
+        sleep 2 ; # 1 second intervals between attempts
+        $MC_COMMAND ;
+        STATUS=$? ;
+      done ;
+      set -e ; # reset `e` as active
+      return 0
+    }
+    
+    # checkBucketExists ($bucket)
+    # Check if the bucket exists, by using the exit code of `mc ls`
+    checkBucketExists() {
+      BUCKET=$1
+      CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1)
+      return $?
+    }
+    
+    # createBucket ($bucket, $policy, $purge)
+    # Ensure bucket exists, purging if asked to
+    createBucket() {
+      BUCKET=$1
+      POLICY=$2
+      PURGE=$3
+      VERSIONING=$4
+    
+      # Purge the bucket, if set & exists
+      # Since PURGE is user input, check explicitly for `true`
+      if [ $PURGE = true ]; then
+        if checkBucketExists $BUCKET ; then
+          echo "Purging bucket '$BUCKET'."
+          set +e ; # don't exit if this fails
+          ${MC} rm -r --force myminio/$BUCKET
+          set -e ; # reset `e` as active
+        else
+          echo "Bucket '$BUCKET' does not exist, skipping purge."
+        fi
+      fi
+    
+      # Create the bucket if it does not exist
+      if ! checkBucketExists $BUCKET ; then
+        echo "Creating bucket '$BUCKET'"
+        ${MC} mb myminio/$BUCKET
+      else
+        echo "Bucket '$BUCKET' already exists."
+      fi
+    
+    
+      # set versioning for bucket
+      if [ ! -z $VERSIONING ] ; then
+        if [ $VERSIONING = true ] ; then
+            echo "Enabling versioning for '$BUCKET'"
+            ${MC} version enable myminio/$BUCKET
+        elif [ $VERSIONING = false ] ; then
+            echo "Suspending versioning for '$BUCKET'"
+            ${MC} version suspend myminio/$BUCKET
+        fi
+      else
+          echo "Bucket '$BUCKET' versioning unchanged."
+      fi
+    
+      # At this point, the bucket should exist, skip checking for existence
+      # Set policy on the bucket
+      echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
+      ${MC} policy set $POLICY myminio/$BUCKET
+    }
+    
+    # Try connecting to Minio instance
+    scheme=http
+    connectToMinio $scheme
+    # Create the buckets
+    createBucket mimir-tsdb none false 
+    createBucket mimir-ruler none false 
+    createBucket enterprise-metrics-tsdb none false 
+    createBucket enterprise-metrics-admin none false 
+    createBucket enterprise-metrics-ruler none false
diff --git a/...test-enterprise-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml b/...test-enterprise-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml
@@ -0,0 +1,72 @@
+---
+# Source: mimir-distributed/charts/minio/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-enterprise-values-minio
+  labels:
+    app: minio
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm
+spec:
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 100%
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app: minio
+      release: test-enterprise-values
+  template:
+    metadata:
+      name: test-enterprise-values-minio
+      labels:
+        app: minio
+        release: test-enterprise-values
+      annotations:
+        checksum/secrets: aac2dea246043210c2649ffaf6c6ea57cd94d0aaf2b21759b8b38a093096478e
+        checksum/config: 8109517e3e9f729fb84cca8217b59099354497df0ca310f4e4b07d4951b02dc4
+    spec:
+      serviceAccountName: "test-enterprise-values-minio"
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: minio
+          image: "minio/minio:RELEASE.2021-02-14T04-01-33Z"
+          imagePullPolicy: IfNotPresent
+          command:
+            - "/bin/sh"
+            - "-ce"
+            - "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server /export"
+          volumeMounts:
+            - name: export
+              mountPath: /export            
+          ports:
+            - name: http
+              containerPort: 9000
+          env:
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: test-enterprise-values-minio
+                  key: accesskey
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: test-enterprise-values-minio
+                  key: secretkey
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi      
+      volumes:
+        - name: export
+          persistentVolumeClaim:
+            claimName: test-enterprise-values-minio
+        - name: minio-user
+          secret:
+            secretName: test-enterprise-values-minio
diff --git a/...es-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml b/...es-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml
@@ -0,0 +1,47 @@
+---
+# Source: mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-enterprise-values-minio-make-bucket-job
+  labels:
+    app: minio-make-bucket-job
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+spec:
+  template:
+    metadata:
+      labels:
+        app: minio-job
+        release: test-enterprise-values
+    spec:
+      restartPolicy: OnFailure      
+      volumes:
+        - name: minio-configuration
+          projected:
+            sources:
+            - configMap:
+                name: test-enterprise-values-minio
+            - secret:
+                name: test-enterprise-values-minio
+      serviceAccountName: "test-enterprise-values-minio"
+      containers:
+      - name: minio-mc
+        image: "minio/mc:RELEASE.2021-02-14T04-28-06Z"
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh", "/config/initialize"]
+        env:
+          - name: MINIO_ENDPOINT
+            value: test-enterprise-values-minio
+          - name: MINIO_PORT
+            value: "9000"
+        volumeMounts:
+          - name: minio-configuration
+            mountPath: /config
+        resources:
+          requests:
+            memory: 128Mi
diff --git a/...erated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml b/...erated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml
@@ -0,0 +1,37 @@
+---
+# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: test-enterprise-values-minio-update-prometheus-secret
+  labels:
+    app: minio-update-prometheus-secret
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+      - patch
+    resourceNames:
+      - test-enterprise-values-minio-prometheus
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+    verbs:
+      - get
+    resourceNames:
+      - test-enterprise-values-minio
diff --git a/...mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml b/...mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml
@@ -0,0 +1,19 @@
+---
+# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: test-enterprise-values-minio-update-prometheus-secret
+  labels:
+    app: minio-update-prometheus-secret
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: test-enterprise-values-minio-update-prometheus-secret
+subjects:
+  - kind: ServiceAccount
+    name: test-enterprise-values-minio-update-prometheus-secret
+    namespace: "default"
diff --git a/...ir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml b/...ir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml
@@ -0,0 +1,11 @@
+---
+# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test-enterprise-values-minio-update-prometheus-secret
+  labels:
+    app: minio-update-prometheus-secret
+    chart: minio-8.0.10
+    release: test-enterprise-values
+    heritage: Helm