[meta] stabilize CI tests (elastic#935)

This PR add some improvements to stabilize CI tests which are flaking at this time:

- watch resources rollout after upgrade to ensure that goss tests when rollout is done
- use newest running pod for goss tests to ensure that it is not using a pod from previous release or a pod which is terminating
- remove mention of hostname in goss tests to ensure that test can run in every pods of the Statefulsets, DaemonSets or Deployments
- increase some more timeouts for Elasticsearch tests
- standardize examples to simplify maintenance and diff

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,7 +18,7 @@ e.g. `helm get elasticsearch` (replace `elasticsearch` with the name of your hel
 
 *Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting.*
 
-*If you find some secrets in plain text in `helm get release` output you should use [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to managed them is a secure way (see [Security Example](https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/security/security.yml#L23-L38)).*
+*If you find some secrets in plain text in `helm get release` output you should use [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to managed them is a secure way (see [Security Example](https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/security/values.yaml#L23-L38)).*
 
 <details>
 <summary>Output of helm get release</summary>

apm-server/examples/default/Makefile

@@ -5,7 +5,7 @@ include ../../../helpers/examples.mk
 RELEASE := helm-apm-server-default
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../
 
 test: install goss
 

apm-server/examples/oss/Makefile

@@ -5,7 +5,7 @@ include ../../../helpers/examples.mk
 RELEASE := helm-apm-server-oss
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install goss
 

apm-server/examples/security/Makefile

@@ -5,7 +5,7 @@ include ../../../helpers/examples.mk
 RELEASE := helm-apm-server-security
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install goss
 

apm-server/examples/upgrade/Makefile

@@ -8,6 +8,7 @@ FROM := 7.6.0	# 7.6.0 is the first release for apm-server
 
 install:
 	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
+	kubectl rollout status deployment $(RELEASE)-apm-server
 
 test: install goss
 

elasticsearch/examples/config/Makefile

@@ -1,10 +1,12 @@
 default: test
+
 include ../../../helpers/examples.mk
 
 RELEASE := helm-es-config
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 secrets:
 	kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack elastic-config-custom-path || true

elasticsearch/examples/config/test/goss.yaml

@@ -3,7 +3,7 @@ http:
     status: 200
     timeout: 2000
     body:
-      - 'green'
+      - "green"
       - '"number_of_nodes":1'
       - '"number_of_data_nodes":1'
 
@@ -12,8 +12,7 @@ http:
     timeout: 2000
     body:
       - '"cluster_name" : "config"'
-      - '"name" : "config-master-0"'
-      - 'You Know, for Search'
+      - "You Know, for Search"
 
 command:
   "elasticsearch-keystore list":

elasticsearch/examples/default/Makefile

@@ -3,12 +3,10 @@ default: test
 include ../../../helpers/examples.mk
 
 RELEASE := helm-es-default
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) ../../
-
-restart:
-	helm upgrade --set terminationGracePeriod=121 --wait --timeout=900s --install $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../
 
 test: install goss
 

elasticsearch/examples/default/test/goss.yaml

@@ -1,13 +1,13 @@
 kernel-param:
   vm.max_map_count:
-    value: '262144'
+    value: "262144"
 
 http:
   http://elasticsearch-master:9200/_cluster/health:
     status: 200
     timeout: 2000
     body:
-      - 'green'
+      - "green"
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
@@ -17,8 +17,7 @@ http:
     body:
       - '"number" : "7.10.0"'
       - '"cluster_name" : "elasticsearch"'
-      - '"name" : "elasticsearch-master-0"'
-      - 'You Know, for Search'
+      - "You Know, for Search"
 
 file:
   /usr/share/elasticsearch/data:

elasticsearch/examples/docker-for-mac/Makefile

@@ -1,9 +1,10 @@
 default: test
 
 RELEASE := helm-es-docker-for-mac
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install
 	helm test $(RELEASE)

elasticsearch/examples/kubernetes-kind/Makefile

@@ -1,13 +1,14 @@
 default: test
 
 RELEASE := helm-es-kind
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 install-local-path:
 	kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml
-	helm upgrade --wait --timeout=1200s --install --values values-local-path.yaml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values-local-path.yaml $(RELEASE) ../../
 
 test: install
 	helm test $(RELEASE)

elasticsearch/examples/microk8s/Makefile

@@ -1,9 +1,10 @@
 default: test
 
 RELEASE := helm-es-microk8s
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install
 	helm test $(RELEASE)

elasticsearch/examples/migration/Makefile

@@ -1,10 +1,10 @@
 PREFIX := helm-es-migration
 
 data:
-	helm upgrade --wait --timeout=900s --install --values ./data.yml $(PREFIX)-data ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values data.yaml $(PREFIX)-data ../../
 
 master:
-	helm upgrade --wait --timeout=900s --install --values ./master.yml $(PREFIX)-master ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values master.yaml $(PREFIX)-master ../../
 
 client:
-	helm upgrade --wait --timeout=900s --install --values ./client.yml $(PREFIX)-client ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values client.yaml $(PREFIX)-client ../../

elasticsearch/examples/migration/README.md

@@ -56,7 +56,7 @@ cluster is healthy before continuing:
   ```
 
 * Deploy new data nodes which will join the existing cluster. Take a look at the
-configuration in [data.yml][]:
+configuration in [data.yaml][]:
 
   ```
   make data
@@ -102,7 +102,7 @@ nodes as to not lose quorum:
   ```
 
 * Now deploy a single new master so that we have 3 masters again. See
-[master.yml][] for the configuration:
+[master.yaml][] for the configuration:
 
   ```
   make master
@@ -114,7 +114,7 @@ nodes as to not lose quorum:
   kubectl scale statefulsets my-release-elasticsearch-master --replicas=1
   ```
 
-* Edit the masters in [masters.yml][] to 2 and redeploy:
+* Edit the masters in [masters.yaml][] to 2 and redeploy:
 
   ```
   make master
@@ -126,7 +126,7 @@ nodes as to not lose quorum:
   kubectl scale statefulsets my-release-elasticsearch-master --replicas=0
   ```
 
-* Edit the [masters.yml][] to have 3 replicas and remove the
+* Edit the [masters.yaml][] to have 3 replicas and remove the
 `discovery.zen.ping.unicast.hosts` entry from `extraEnvs` then redeploy the
 masters. This will make sure all 3 masters are running in the new cluster and
 are pointing at each other for discovery:
@@ -160,8 +160,8 @@ client nodes:
 working correctly you can cleanup leftover resources from your old cluster.
 
 [basic license]: https://www.elastic.co/subscriptions
-[data.yml]: https://github.com/elastic/helm-charts/blob/7.10/elasticsearch/examples/migration/data.yml
-[helm/charts]: https://github.com/helm/charts/tree/master/stable/elasticsearch
-[master.yml]: https://github.com/elastic/helm-charts/blob/7.10/elasticsearch/examples/migration/master.yml
+[data.yaml]: https://github.com/elastic/helm-charts/blob/7.10/elasticsearch/examples/migration/data.yaml
+[helm/charts]: https://github.com/helm/charts/tree/7.10/stable/elasticsearch
+[master.yaml]: https://github.com/elastic/helm-charts/blob/7.10/elasticsearch/examples/migration/master.yaml
 [restoring to a different cluster guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-snapshots.html#_restoring_to_a_different_cluster
 [rolling upgrades guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html

elasticsearch/examples/minikube/Makefile

@@ -1,9 +1,10 @@
 default: test
 
 RELEASE := helm-es-minikube
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install
 	helm test $(RELEASE)

elasticsearch/examples/multi/Makefile

@@ -4,11 +4,12 @@ include ../../../helpers/examples.mk
 
 PREFIX := helm-es-multi
 RELEASE := helm-es-multi-master
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=900s --install --values ./master.yml $(PREFIX)-master ../../
-	helm upgrade --wait --timeout=900s --install --values ./data.yml $(PREFIX)-data ../../
-	helm upgrade --wait --timeout=900s --install --values ./client.yml $(PREFIX)-client ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values master.yaml $(PREFIX)-master ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values data.yaml $(PREFIX)-data ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values client.yaml $(PREFIX)-client ../../
 
 test: install goss
 

elasticsearch/examples/multi/README.md

@@ -23,7 +23,7 @@ releases:
 You can also run [goss integration tests][] using `make test`
 
 
-[client values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/client.yml
-[data values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/data.yml
+[client values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/client.yaml
+[data values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/data.yaml
 [goss integration tests]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/test/goss.yaml
-[master values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/master.yml
+[master values]: https://github.com/elastic/helm-charts/tree/7.10/elasticsearch/examples/multi/master.yaml

elasticsearch/examples/openshift/Makefile

@@ -1,13 +1,11 @@
 default: test
+
 include ../../../helpers/examples.mk
 
 RELEASE := elasticsearch
 
-template:
-	helm template --values ./values.yaml ../../
-
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install goss
 

elasticsearch/examples/openshift/test/goss.yaml

@@ -3,7 +3,7 @@ http:
     status: 200
     timeout: 2000
     body:
-      - 'green'
+      - "green"
       - '"number_of_nodes":3'
       - '"number_of_data_nodes":3'
 
@@ -13,5 +13,4 @@ http:
     body:
       - '"number" : "7.10.0"'
       - '"cluster_name" : "elasticsearch"'
-      - '"name" : "elasticsearch-master-0"'
-      - 'You Know, for Search'
+      - "You Know, for Search"

elasticsearch/examples/oss/Makefile

@@ -1,10 +1,12 @@
 default: test
+
 include ../../../helpers/examples.mk
 
 RELEASE := helm-es-oss
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
 
 test: install goss
 

elasticsearch/examples/oss/test/goss.yaml

@@ -13,5 +13,4 @@ http:
     body:
       - '"number" : "7.10.0"'
       - '"cluster_name" : "oss"'
-      - '"name" : "oss-master-0"'
       - 'You Know, for Search'

elasticsearch/examples/security/Makefile

@@ -4,16 +4,17 @@ include ../../../helpers/examples.mk
 
 RELEASE := helm-es-security
 ELASTICSEARCH_IMAGE := docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION)
+TIMEOUT := 1200s
 
 install:
-	helm upgrade --wait --timeout=900s --install --values ./security.yml $(RELEASE) ../../
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
+
+test: secrets install goss
 
 purge:
 	kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem elastic-certificate-crt|| true
 	helm del $(RELEASE)
 
-test: secrets install goss
-
 pull-elasticsearch-image:
 	docker pull $(ELASTICSEARCH_IMAGE)
 
@@ -33,5 +34,5 @@ secrets:
 	kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \
 	kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \
 	kubectl create secret generic elastic-certificate-crt --from-file=elastic-certificate.crt && \
-	kubectl create secret generic elastic-credentials  --from-literal=password=$$password --from-literal=username=elastic && \
+	kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \
 	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12