Merge pull request #1463 from elastic/statefulset-refactoring

Merge the statefulset-refactoring feature branch into master.

Change the way Elasticsearch nodes are orchestrated to rely on StatefulSets instead of managing pods.

The implementation is not complete yet. In particular:

it ignores any change budget considerations
if a StatefulSet needs to be replaced, the entire new sset will be added (with all replicas) before the old one is shut down (budget ignored)
it does not support grow and shrink, but does rolling upgrades by default
it does not remove PVCs, which can be a problem when creating/deleting multiple times the same stack: data volumes from the first stack will be reused for the second one
it probably has some orchestration bugs :)
it is missing a bit of refactoring and unit testing

However, it seems to be good enough to spawn stacks, apply any mutation, and run E2E tests.

docs/index.asciidoc

@@ -15,7 +15,3 @@ include::snapshots.asciidoc[]
 include::elasticsearch-spec.asciidoc[]
 include::apm.asciidoc[]
 include::troubleshooting.asciidoc[]
-
-
-
-

operators/config/crds/elasticsearch_v1alpha1_elasticsearch.yaml

@@ -120,10 +120,11 @@ spec:
                     description: Config represents Elasticsearch configuration.
                     type: object
                   name:
-                    description: Name is a logical name for this set of nodes. Used
-                      as a part of the managed Elasticsearch node.name setting.
-                    maxLength: 12
-                    pattern: '[a-zA-Z0-9-]*'
+                    description: 'Name is a logical name for this set of nodes. Used
+                      as a part of the managed Elasticsearch node.name setting. TODO:
+                      refactor and explain name length conventions'
+                    maxLength: 19
+                    pattern: '[a-zA-Z0-9-]+'
                     type: string
                   nodeCount:
                     description: NodeCount defines how many nodes have this topology
@@ -147,6 +148,8 @@ spec:
                     items:
                       type: object
                     type: array
+                required:
+                - name
                 type: object
               type: array
             podDisruptionBudget:

operators/config/e2e/global_operator.yaml

@@ -39,6 +39,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - get
   - list

operators/config/e2e/namespace_operator.yaml

@@ -47,6 +47,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - get
   - list

operators/config/operator/all-in-one/cluster_role.template.yaml

@@ -29,6 +29,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - get
   - list

operators/config/operator/global/cluster_role.template.yaml

@@ -28,6 +28,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - get
   - list

operators/config/operator/namespace/cluster_role.template.yaml

@@ -41,6 +41,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - get
   - list

operators/config/samples/apm/apm_es_kibana.yaml

@@ -3,28 +3,29 @@
 apiVersion: elasticsearch.k8s.elastic.co/v1alpha1
 kind: Elasticsearch
 metadata:
-  name: elasticsearch-sample
+  name: es-apm-sample
 spec:
   version: 7.2.0
   nodes:
-  - nodeCount: 3
+  - name: default
+    nodeCount: 3
 ---
 apiVersion: apm.k8s.elastic.co/v1alpha1
 kind: ApmServer
 metadata:
-  name: apm-server-sample
+  name: apm-apm-sample
 spec:
   version: 7.2.0
   nodeCount: 1
   elasticsearchRef:
-    name: "elasticsearch-sample"
+    name: "es-apm-sample"
 ---
 apiVersion: kibana.k8s.elastic.co/v1alpha1
 kind: Kibana
 metadata:
-  name: kibana-sample
+  name: kb-apm-sample
 spec:
   version: 7.2.0
   nodeCount: 1
   elasticsearchRef:
-    name: "elasticsearch-sample"
+    name: "es-apm-sample"

operators/config/samples/elasticsearch/elasticsearch.yaml

@@ -6,7 +6,8 @@ metadata:
 spec:
   version: 7.2.0
   nodes:
-  - config:
+  - name: default
+    config:
       # most Elasticsearch configuration parameters are possible to set, e.g:
       node.attr.attr_name: attr_value
       node.master: true

operators/config/samples/kibana/kibana_es.yaml

@@ -6,7 +6,8 @@ metadata:
 spec:
   version: 7.2.0
   nodes:
-  - nodeCount: 1
+  - name: default
+    nodeCount: 1
 ---
 apiVersion: kibana.k8s.elastic.co/v1alpha1
 kind: Kibana

operators/pkg/apis/elasticsearch/v1alpha1/elasticsearch_types.go

@@ -68,9 +68,10 @@ func (es ElasticsearchSpec) NodeCount() int32 {
 // NodeSpec defines a common topology for a set of Elasticsearch nodes
 type NodeSpec struct {
 	// Name is a logical name for this set of nodes. Used as a part of the managed Elasticsearch node.name setting.
-	// +kubebuilder:validation:Pattern=[a-zA-Z0-9-]*
-	// +kubebuilder:validation:MaxLength=12
-	Name string `json:"name,omitempty"`
+	// +kubebuilder:validation:Pattern=[a-zA-Z0-9-]+
+	// +kubebuilder:validation:MaxLength=19
+	// TODO: refactor and explain name length conventions
+	Name string `json:"name"`
 
 	// Config represents Elasticsearch configuration.
 	Config *commonv1alpha1.Config `json:"config,omitempty"`

operators/pkg/controller/apmserver/apmserver_controller_test.go

@@ -10,6 +10,7 @@ import (
 	apmv1alpha1 "github.com/elastic/cloud-on-k8s/operators/pkg/apis/apm/v1alpha1"
 	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/certificates/http"
+	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/defaults"
 	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/keystore"
 	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/watches"
 	"github.com/elastic/cloud-on-k8s/operators/pkg/utils/k8s"
@@ -69,25 +70,7 @@ func (tp testParams) withInitContainer() testParams {
 			},
 			Name:  "",
 			Image: "docker.elastic.co/apm/apm-server:1.0",
-			Env: []corev1.EnvVar{{
-				Name: "POD_NAME",
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						APIVersion: "v1",
-						FieldPath:  "metadata.name",
-					},
-				},
-			},
-				{
-					Name: "POD_IP",
-					ValueFrom: &corev1.EnvVarSource{
-						FieldRef: &corev1.ObjectFieldSelector{
-							APIVersion: "v1",
-							FieldPath:  "status.podIP",
-						},
-					},
-				},
-			},
+			Env:   defaults.PodDownwardEnvVars,
 		},
 	}
 	return tp
@@ -137,7 +120,6 @@ func expectedDeploymentParams() testParams {
 				},
 				Containers: []corev1.Container{{
 					VolumeMounts: []corev1.VolumeMount{
-
 						{
 							Name:      "config",
 							ReadOnly:  true,
@@ -163,27 +145,17 @@ func expectedDeploymentParams() testParams {
 						"-c",
 						"config/config-secret/apm-server.yml",
 					},
-					Env: []corev1.EnvVar{{
-						Name: "POD_NAME",
+					Env: append(defaults.PodDownwardEnvVars, corev1.EnvVar{
+						Name: "SECRET_TOKEN",
 						ValueFrom: &corev1.EnvVarSource{
-							FieldRef: &corev1.ObjectFieldSelector{
-								APIVersion: "v1",
-								FieldPath:  "metadata.name",
-							},
-						},
-					},
-						{
-							Name: "SECRET_TOKEN",
-							ValueFrom: &corev1.EnvVarSource{
-								SecretKeyRef: &corev1.SecretKeySelector{
-									LocalObjectReference: corev1.LocalObjectReference{
-										Name: "test-apm-server-apm-token",
-									},
-									Key: "secret-token",
+							SecretKeyRef: &corev1.SecretKeySelector{
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: "test-apm-server-apm-token",
 								},
+								Key: "secret-token",
 							},
 						},
-					},
+					}),
 					Ports: []corev1.ContainerPort{
 						{Name: "http", ContainerPort: int32(8200), Protocol: corev1.ProtocolTCP},
 					},

operators/pkg/controller/apmserver/pod.go

@@ -87,23 +87,15 @@ func newPodSpec(as *v1alpha1.ApmServer, p PodSpecParams) corev1.PodTemplateSpec
 		filepath.Join(ConfigVolumePath, "config-secret"),
 	)
 
-	env := []corev1.EnvVar{
-		{
-			Name: "POD_NAME",
-			ValueFrom: &corev1.EnvVarSource{
-				FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"},
+	env := append(defaults.PodDownwardEnvVars, corev1.EnvVar{
+		Name: "SECRET_TOKEN",
+		ValueFrom: &corev1.EnvVarSource{
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{Name: p.ApmServerSecret.Name},
+				Key:                  SecretTokenKey,
 			},
 		},
-		{
-			Name: "SECRET_TOKEN",
-			ValueFrom: &corev1.EnvVarSource{
-				SecretKeyRef: &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{Name: p.ApmServerSecret.Name},
-					Key:                  SecretTokenKey,
-				},
-			},
-		},
-	}
+	})
 
 	builder := defaults.NewPodTemplateBuilder(
 		p.PodTemplate, v1alpha1.APMServerContainerName).

operators/pkg/controller/apmserver/pod_test.go

@@ -8,10 +8,12 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/elastic/cloud-on-k8s/operators/pkg/apis/apm/v1alpha1"
-	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/volume"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/elastic/cloud-on-k8s/operators/pkg/apis/apm/v1alpha1"
+	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/common/volume"
+	"github.com/elastic/cloud-on-k8s/operators/pkg/controller/elasticsearch/settings"
 )
 
 func TestNewPodSpec(t *testing.T) {
@@ -64,6 +66,12 @@ func TestNewPodSpec(t *testing.T) {
 							Name:  v1alpha1.APMServerContainerName,
 							Image: imageWithVersion(defaultImageRepositoryAndName, "7.0.1"),
 							Env: []corev1.EnvVar{
+								{
+									Name: settings.EnvPodIP,
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIP"},
+									},
+								},
 								{
 									Name: "POD_NAME",
 									ValueFrom: &corev1.EnvVarSource{