[Kibana] 8.5.1"message":"secrets \"kibana-kibana-es-token\" already exists"

[JoranDox]

Chart version:
8.5.1

Kubernetes version:
GitVersion:"v1.23.8"

Kubernetes provider:
Azure AKS

Helm Version:
Version:"v3.10.3"

helm get release output

Maybe the instructions here should be updated to ask for helm get all ${releasename}, because it was really unclear what to do, and that release was the name of a release and not a keyword.

Output of helm get release

NAME: kibana
LAST DEPLOYED: Thu Feb  2 09:16:19 2023
NAMESPACE: <redacted>
STATUS: failed
REVISION: 2
TEST SUITE: None
USER-SUPPLIED VALUES:
null

COMPUTED VALUES:
affinity: {}
annotations: {}
automountToken: true
elasticsearchCertificateAuthoritiesFile: ca.crt
elasticsearchCertificateSecret: elasticsearch-master-certs
elasticsearchCredentialSecret: elasticsearch-master-credentials
elasticsearchHosts: https://elasticsearch-master:9200
envFrom: []
extraContainers: []
extraEnvs:
- name: NODE_OPTIONS
  value: --max-old-space-size=1800
extraInitContainers: []
extraVolumeMounts: []
extraVolumes: []
fullnameOverride: ""
healthCheckPath: /app/kibana
hostAliases: []
httpPort: 5601
image: docker.elastic.co/kibana/kibana
imagePullPolicy: IfNotPresent
imagePullSecrets: []
imageTag: 8.5.1
ingress:
  annotations: {}
  className: nginx
  enabled: false
  hosts:
  - host: kibana-example.local
    paths:
    - path: /
  pathtype: ImplementationSpecific
kibanaConfig: {}
labels: {}
lifecycle: {}
nameOverride: ""
nodeSelector: {}
podAnnotations: {}
podSecurityContext:
  fsGroup: 1000
priorityClassName: ""
protocol: http
readinessProbe:
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 3
  timeoutSeconds: 5
replicas: 1
resources:
  limits:
    cpu: 1000m
    memory: 2Gi
  requests:
    cpu: 1000m
    memory: 2Gi
secretMounts: []
securityContext:
  capabilities:
    drop:
    - ALL
  runAsNonRoot: true
  runAsUser: 1000
serverHost: 0.0.0.0
service:
  annotations: {}
  httpPortName: http
  labels: {}
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  nodePort: ""
  port: 5601
  type: ClusterIP
serviceAccount: ""
tolerations: []
updateStrategy:
  type: Recreate

HOOKS:
---
# Source: kibana/templates/post-delete-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: post-delete-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": post-delete
    "helm.sh/hook-delete-policy": hook-succeeded
---
# Source: kibana/templates/pre-install-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: pre-install-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade
    "helm.sh/hook-delete-policy": hook-succeeded
---
# Source: kibana/templates/configmap-helm-scripts.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: kibana-kibana-helm-scripts
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade,post-delete
    "helm.sh/hook-delete-policy": hook-succeeded
data:
  manage-es-token.js: |
    const https = require('https');
    const fs = require('fs');

    // Read environment variables
    function getEnvVar(name) {
      if (!process.env[name]) {
        throw new Error(name + ' environment variable is missing')
      }
      return process.env[name]
    }

    // Elasticsearch API
    const esPath = '_security/service/elastic/kibana/credential/token/kibana-kibana';
    const esUrl = 'https://elasticsearch-master:9200' + '/' + esPath
    const esUsername = getEnvVar('ELASTICSEARCH_USERNAME');
    const esPassword = getEnvVar('ELASTICSEARCH_PASSWORD');
    const esAuth = esUsername + ':' + esPassword;
    const esCaFile = getEnvVar('ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES');
    const esCa = fs.readFileSync(esCaFile);

    // Kubernetes API
    const k8sHostname = getEnvVar('KUBERNETES_SERVICE_HOST');
    const k8sPort = getEnvVar('KUBERNETES_SERVICE_PORT_HTTPS');
    const k8sPostSecretPath = 'api/v1/namespaces/<redacted>/secrets';
    const k8sDeleteSecretPath = 'api/v1/namespaces/<redacted>/secrets/kibana-kibana-es-token';
    const k8sPostSecretUrl = `https://${k8sHostname}:${k8sPort}/${k8sPostSecretPath}`;
    const k8sDeleteSecretUrl = `https://${k8sHostname}:${k8sPort}/${k8sDeleteSecretPath}`;
    const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
    const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');

    // Post Data
    const esTokenDeleteOptions = {
      method: 'DELETE',
      auth: esAuth,
      ca: esCa,
    };
    const esTokenCreateOptions = {
      method: 'POST',
      auth: esAuth,
      ca: esCa,
    };
    const secretCreateOptions = {
      method: 'POST',
      ca: k8sCa,
      headers: {
        'Authorization': 'Bearer ' + k8sBearer,
        'Accept': 'application/json',
        'Content-Type': 'application/json',
      }
    };
    const secretDeleteOptions = {
      method: 'DELETE',
      ca: k8sCa,
      headers: {
        'Authorization': 'Bearer ' + k8sBearer,
        'Accept': 'application/json',
        'Content-Type': 'application/json',
      }
    };

    // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
    function requestPromise(url, httpsOptions, extraOptions = {}) {
      return new Promise((resolve, reject) => {
        const request = https.request(url, httpsOptions, response => {

          console.log('statusCode:', response.statusCode);

          let isSuccess = undefined;

          if (typeof(extraOptions.extraStatusCode) != "undefined" && extraOptions.extraStatusCode != null) {
            isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == extraOptions.extraStatusCode;
          } else {
            isSuccess = response.statusCode >= 200 && response.statusCode < 300;
          }

          let data = '';
          response.on('data', chunk => data += chunk); // accumulate data
          response.once('end', () => isSuccess ? resolve(data) : reject(data));  // resolve promise here
        });

        request.once('error', err => {
          // This won't log anything for e.g. an HTTP 404 or 500 response,
          // since from HTTP's point-of-view we successfully received a
          // response.
          console.log(`${httpsOptions.method} ${httpsOptions.path} failed: `, err.message || err);
          reject(err);  // if promise is not already resolved, then we can reject it here
        });

        if (typeof(extraOptions.payload) != "undefined") {
          request.write(extraOptions.payload);
        }
        request.end();
      });
    }

    function createEsToken() {
      // Chaining requests
      console.log('Cleaning previous token');
      // 404 status code is accepted if there is no previous token to clean
      return requestPromise(esUrl, esTokenDeleteOptions, {extraStatusCode: 404}).then(() => {
        console.log('Creating new token');
        return requestPromise(esUrl, esTokenCreateOptions).then(response => {
          const body = JSON.parse(response);
          const token = body.token.value

          // Encode the token in base64
          const base64Token = Buffer.from(token, 'utf8').toString('base64');

          // Prepare the k8s secret
          const secretData = JSON.stringify({
            "apiVersion": "v1",
            "kind": "Secret",
            "metadata": {
              "namespace": "<redacted>",
              "name": "kibana-kibana-es-token",
            },
            "type": "Opaque",
            "data": {
              "token": base64Token,
            }
          })

          // Create the k8s secret
          console.log('Creating K8S secret');
          return requestPromise(k8sPostSecretUrl, secretCreateOptions, {payload: secretData})
        });
      });
    }

    function cleanEsToken() {
      // Chaining requests
      console.log('Cleaning token');
      return requestPromise(esUrl, esTokenDeleteOptions).then(() => {
        // Create the k8s secret
        console.log('Delete K8S secret');
        return requestPromise(k8sDeleteSecretUrl, secretDeleteOptions)
      });
    }

    const command = process.argv[2];
    switch (command) {
      case 'create':
        console.log('Creating a new Elasticsearch token for Kibana')
        createEsToken().catch(err => {
          console.error(err);
          process.exit(1);
        });
        break;
      case 'clean':
        console.log('Cleaning the Kibana Elasticsearch token')
        cleanEsToken().catch(err => {
          console.error(err);
          process.exit(1);
        });
        break;
      default:
        console.log('Unknown command');
        process.exit(1);
    }
---
# Source: kibana/templates/post-delete-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: post-delete-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": post-delete
    "helm.sh/hook-delete-policy": hook-succeeded
rules:
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - delete
---
# Source: kibana/templates/pre-install-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pre-install-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade
    "helm.sh/hook-delete-policy": hook-succeeded
rules:
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - create
      - update
---
# Source: kibana/templates/post-delete-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: post-delete-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": post-delete
    "helm.sh/hook-delete-policy": hook-succeeded
subjects:
  - kind: ServiceAccount
    name: post-delete-kibana-kibana
    namespace: "<redacted>"
roleRef:
  kind: Role
  name: post-delete-kibana-kibana
  apiGroup: rbac.authorization.k8s.io
---
# Source: kibana/templates/pre-install-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: pre-install-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade
    "helm.sh/hook-delete-policy": hook-succeeded
subjects:
  - kind: ServiceAccount
    name: pre-install-kibana-kibana
    namespace: "<redacted>"
roleRef:
  kind: Role
  name: pre-install-kibana-kibana
  apiGroup: rbac.authorization.k8s.io
---
# Source: kibana/templates/post-delete-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: post-delete-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": post-delete
    "helm.sh/hook-delete-policy": hook-succeeded
spec:
  backoffLimit: 3
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: clean-kibana-token
          image: "docker.elastic.co/kibana/kibana:8.5.1"
          imagePullPolicy: "IfNotPresent"
          command: ["/usr/share/kibana/node/bin/node"]
          args:
           - /usr/share/kibana/helm-scripts/manage-es-token.js
           - clean
          env:
            - name: "ELASTICSEARCH_USERNAME"
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-master-credentials
                  key: username
            - name: "ELASTICSEARCH_PASSWORD"
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-master-credentials
                  key: password
            - name: ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES
              value: "/usr/share/kibana/config/certs/ca.crt"
          volumeMounts:
            - name: elasticsearch-certs
              mountPath: /usr/share/kibana/config/certs
              readOnly: true
            - name: kibana-helm-scripts
              mountPath: /usr/share/kibana/helm-scripts
      serviceAccount: post-delete-kibana-kibana
      volumes:
        - name: elasticsearch-certs
          secret:
            secretName: elasticsearch-master-certs
        - name: kibana-helm-scripts
          configMap:
            name: kibana-kibana-helm-scripts
            defaultMode: 0755
---
# Source: kibana/templates/pre-install-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: pre-install-kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade
    "helm.sh/hook-delete-policy": hook-succeeded
spec:
  backoffLimit: 20
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: create-kibana-token
          image: "docker.elastic.co/kibana/kibana:8.5.1"
          imagePullPolicy: "IfNotPresent"
          command: ["/usr/share/kibana/node/bin/node"]
          args:
           - /usr/share/kibana/helm-scripts/manage-es-token.js
           - create
          env:
            - name: "ELASTICSEARCH_USERNAME"
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-master-credentials
                  key: username
            - name: "ELASTICSEARCH_PASSWORD"
              valueFrom:
                secretKeyRef:
                  name: elasticsearch-master-credentials
                  key: password
            - name: ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES
              value: "/usr/share/kibana/config/certs/ca.crt"
          volumeMounts:
            - name: elasticsearch-certs
              mountPath: /usr/share/kibana/config/certs
              readOnly: true
            - name: kibana-helm-scripts
              mountPath: /usr/share/kibana/helm-scripts
      serviceAccount: pre-install-kibana-kibana
      volumes:
        - name: elasticsearch-certs
          secret:
            secretName: elasticsearch-master-certs
        - name: kibana-helm-scripts
          configMap:
            name: kibana-kibana-helm-scripts
            defaultMode: 0755
MANIFEST:
---
# Source: kibana/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
spec:
  type: ClusterIP
  ports:
    - port: 5601
      protocol: TCP
      name: http
      targetPort: 5601
  selector:
    app: kibana
    release: "kibana"
---
# Source: kibana/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana-kibana
  labels: 
    app: kibana
    release: "kibana"
    heritage: Helm
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: kibana
      release: "kibana"
  template:
    metadata:
      labels:
        app: kibana
        release: "kibana"
      annotations:
        
    spec:
      automountServiceAccountToken: true
      securityContext:
        fsGroup: 1000
      volumes:
        - name: kibana-tokens
          emptyDir: {}
        - name: elasticsearch-certs
          secret:
            secretName: elasticsearch-master-certs
      initContainers:
      containers:
      - name: kibana
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
        image: "docker.elastic.co/kibana/kibana:8.5.1"
        imagePullPolicy: "IfNotPresent"
        env:
          - name: ELASTICSEARCH_HOSTS
            value: "https://elasticsearch-master:9200"
          - name: ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES
            value: "/usr/share/kibana/config/certs/ca.crt"
          - name: SERVER_HOST
            value: "0.0.0.0"
          - name: ELASTICSEARCH_SERVICEACCOUNTTOKEN
            valueFrom:
              secretKeyRef:
                name: kibana-kibana-es-token
                key: token
                optional: false
          - name: NODE_OPTIONS
            value: --max-old-space-size=1800
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
          exec:
            command:
              - bash
              - -c
              - |
                #!/usr/bin/env bash -e

                # Disable nss cache to avoid filling dentry cache when calling curl
                # This is required with Kibana Docker using nss < 3.52
                export NSS_SDB_USE_CACHE=no

                http () {
                    local path="${1}"
                    set -- -XGET -s --fail -L

                    if [ -n "${ELASTICSEARCH_USERNAME}" ] && [ -n "${ELASTICSEARCH_PASSWORD}" ]; then
                      set -- "$@" -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}"
                    fi

                    STATUS=$(curl --output /dev/null --write-out "%{http_code}" -k "$@" "http://localhost:5601${path}")
                    if [[ "${STATUS}" -eq 200 ]]; then
                      exit 0
                    fi

                    echo "Error: Got HTTP code ${STATUS} but expected a 200"
                    exit 1
                }

                http "/app/kibana"
        ports:
        - containerPort: 5601
        resources:
          limits:
            cpu: 1000m
            memory: 2Gi
          requests:
            cpu: 1000m
            memory: 2Gi
        volumeMounts:
          - name: elasticsearch-certs
            mountPath: /usr/share/kibana/config/certs
            readOnly: true
          - name: kibana-tokens
            mountPath: /usr/share/kibana/config/tokens
            readOnly: true

NOTES:
1. Watch all containers come up.
  $ kubectl get pods --namespace=<redacted> -l release=kibana -w
2. Retrieve the elastic user's password.
  $ kubectl get secrets --namespace=<redacted> elasticsearch-master-credentials -ojsonpath='{.data.password}' | base64 -d
3. Retrieve the kibana service account token.
  $ kubectl get secrets --namespace=<redacted> kibana-kibana-es-token -ojsonpath='{.data.token}' | base64 -d

Describe the bug:
helm upgrade --install kibana elastic/kibana works only once, updating kibana doesn't work

Steps to reproduce:

1. helm upgrade --install elastic elastic/elastic # elastic works
2. helm upgrade --install kibana elastic/kibana # now kibana works, connected to elastic, all fine
3. helm upgrade --install kibana elastic/kibana # times out, and there's a bunch of pods named pre-install-kibana-kibana-XXXXX all in the erred state

Expected behavior:
helm upgrade --install just works and applies new values if relevant

Provide logs and/or server output (if relevant):

Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting

kubectl logs pre-install-kibana-kibana-26xrr

Creating a new Elasticsearch token for Kibana
Cleaning previous token
statusCode: 200
Creating new token
statusCode: 200
Creating K8S secret
statusCode: 409
{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"secrets \"kibana-kibana-es-token\" already exists","reason":"AlreadyExists","details":{"name":"kibana-kibana-es-token","kind":"secrets"},"code":409}

Any additional context:
Looks like it can't handle the token already existing? I see code claiming to clean the token, but I guess that's not working.
Possibly related to #1765 but they didn't add enough output to be sure, so I don't want to assume it's the same issue.

[nattawitc]

I see code claiming to clean the token, but I guess that's not working.

It's working but that's the part where it remove the old token from elasticsearch not from k8s cluster.
The actual problem is from line below where it make a post request to k8s to create a secret. The quick fix is to remove the old secret first

          return requestPromise(k8sDeleteSecretUrl, secretDeleteOptions, {extraStatusCode: 404}).then(() => {
            return requestPromise(k8sPostSecretUrl, secretCreateOptions, {payload: secretData})
          })

That fix the initial problem but it still doesn't work seamlessly if the update doesn't change the deployment. Its pods will not restart and will use the old token. You'll still need to manually restart the deployment.

[bcouetil]

Hello, came here to solve this annoying issue when upgrading.

Thanks for the tips to delete the token.

Will there be any fix soon ?

[doverk96]

@JoranDox Tried installing the Kibana post Elasticsearch deployment but it fails saying cannot use elastic username as it is a superuser. Any help ? how can we create a user for Kibana so that pre install job gets succeeded ?

[djerfy]

Hello, same problem here (with ArgoCD). With Helm, you can upgrade with --no-hooks option