Able to configure custom env for components (#2052) (#2058)

* Able to configure custom env for components

* codegen

Co-authored-by: Yecheng Fu <fuyecheng@pingcap.com>

docs/api-references/docs.html

@@ -2606,6 +2606,34 @@ <h3 id="pingcap.com/v1alpha1.ComponentSpec">ComponentSpec
 Optional: Defaults to cluster-level setting</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>env</code></br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#envvar-v1-core">
+[]Kubernetes core/v1.EnvVar
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>List of environment variables to set in the container, like
+v1.Container.Env.
+Note that following env names cannot be used and may be overrided by
+tidb-operator built envs.
+- NAMESPACE
+- TZ
+- SERVICE_NAME
+- PEER_SERVICE_NAME
+- HEADLESS_SERVICE_NAME
+- SET_NAME
+- HOSTNAME
+- CLUSTER_NAME
+- POD_NAME
+- BINLOG_ENABLED
+- SLOW_LOG_FILE</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="pingcap.com/v1alpha1.ConfigUpdateStrategy">ConfigUpdateStrategy

pkg/apis/pingcap/v1alpha1/tidbcluster_component.go

@@ -36,6 +36,7 @@ type ComponentAccessor interface {
 	DnsPolicy() corev1.DNSPolicy
 	ConfigUpdateStrategy() ConfigUpdateStrategy
 	BuildPodSpec() corev1.PodSpec
+	Env() []corev1.EnvVar
 }
 
 type componentAccessorImpl struct {
@@ -160,6 +161,10 @@ func (a *componentAccessorImpl) BuildPodSpec() corev1.PodSpec {
 	return spec
 }
 
+func (a *componentAccessorImpl) Env() []corev1.EnvVar {
+	return a.ComponentSpec.Env
+}
+
 // BaseTiDBSpec returns the base spec of TiDB servers
 func (tc *TidbCluster) BaseTiDBSpec() ComponentAccessor {
 	return &componentAccessorImpl{&tc.Spec, &tc.Spec.TiDB.ComponentSpec}

pkg/apis/pingcap/v1alpha1/types.go

@@ -455,6 +455,24 @@ type ComponentSpec struct {
 	// Optional: Defaults to cluster-level setting
 	// +optional
 	ConfigUpdateStrategy *ConfigUpdateStrategy `json:"configUpdateStrategy,omitempty"`
+
+	// List of environment variables to set in the container, like
+	// v1.Container.Env.
+	// Note that following env names cannot be used and may be overrided by
+	// tidb-operator built envs.
+	// - NAMESPACE
+	// - TZ
+	// - SERVICE_NAME
+	// - PEER_SERVICE_NAME
+	// - HEADLESS_SERVICE_NAME
+	// - SET_NAME
+	// - HOSTNAME
+	// - CLUSTER_NAME
+	// - POD_NAME
+	// - BINLOG_ENABLED
+	// - SLOW_LOG_FILE
+	// +optional
+	Env []corev1.EnvVar `json:"env,omitempty"`
 }
 
 // +k8s:openapi-gen=true

pkg/apis/pingcap/v1alpha1/validation/validation.go

@@ -20,7 +20,9 @@ import (
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/label"
+	corev1 "k8s.io/api/core/v1"
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
@@ -31,10 +33,151 @@ func ValidateTidbCluster(tc *v1alpha1.TidbCluster) field.ErrorList {
 	// validate metadata
 	fldPath := field.NewPath("metadata")
 	// validate metadata/annotations
-	allErrs = append(allErrs, apivalidation.ValidateAnnotations(tc.ObjectMeta.Annotations, fldPath.Child("annotations"))...)
+	allErrs = append(allErrs, validateAnnotations(tc.ObjectMeta.Annotations, fldPath.Child("annotations"))...)
+	// validate spec
+	allErrs = append(allErrs, validateTiDBClusterSpec(&tc.Spec, field.NewPath("spec"))...)
+	return allErrs
+}
+
+func validateAnnotations(anns map[string]string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, apivalidation.ValidateAnnotations(anns, fldPath)...)
 	for _, key := range []string{label.AnnPDDeleteSlots, label.AnnTiDBDeleteSlots, label.AnnTiKVDeleteSlots} {
-		allErrs = append(allErrs, validateDeleteSlots(tc.ObjectMeta.Annotations, key, fldPath.Child("annotations", key))...)
+		allErrs = append(allErrs, validateDeleteSlots(anns, key, fldPath.Child(key))...)
+	}
+	return allErrs
+}
+
+func validateTiDBClusterSpec(spec *v1alpha1.TidbClusterSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validatePDSpec(&spec.PD, fldPath.Child("pd"))...)
+	allErrs = append(allErrs, validateTiKVSpec(&spec.TiKV, fldPath.Child("tikv"))...)
+	allErrs = append(allErrs, validateTiDBSpec(&spec.TiDB, fldPath.Child("tidb"))...)
+	if spec.Pump != nil {
+		allErrs = append(allErrs, validatePumpSpec(spec.Pump, fldPath.Child("pump"))...)
+	}
+	return allErrs
+}
+
+func validatePDSpec(spec *v1alpha1.PDSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...)
+	return allErrs
+}
+
+func validateTiKVSpec(spec *v1alpha1.TiKVSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...)
+	return allErrs
+}
+
+func validateTiDBSpec(spec *v1alpha1.TiDBSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...)
+	return allErrs
+}
+
+func validatePumpSpec(spec *v1alpha1.PumpSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...)
+	return allErrs
+}
+
+func validateComponentSpec(spec *v1alpha1.ComponentSpec, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	// TODO validate other fields
+	allErrs = append(allErrs, validateEnv(spec.Env, fldPath.Child("env"))...)
+	return allErrs
+}
+
+// validateEnv validates env vars
+func validateEnv(vars []corev1.EnvVar, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	for i, ev := range vars {
+		idxPath := fldPath.Index(i)
+		if len(ev.Name) == 0 {
+			allErrs = append(allErrs, field.Required(idxPath.Child("name"), ""))
+		} else {
+			for _, msg := range validation.IsEnvVarName(ev.Name) {
+				allErrs = append(allErrs, field.Invalid(idxPath.Child("name"), ev.Name, msg))
+			}
+		}
+		allErrs = append(allErrs, validateEnvVarValueFrom(ev, idxPath.Child("valueFrom"))...)
+	}
+	return allErrs
+}
+
+func validateEnvVarValueFrom(ev corev1.EnvVar, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if ev.ValueFrom == nil {
+		return allErrs
+	}
+
+	numSources := 0
+
+	if ev.ValueFrom.FieldRef != nil {
+		numSources++
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("fieldRef"), "", "fieldRef is not supported"))
+	}
+	if ev.ValueFrom.ResourceFieldRef != nil {
+		numSources++
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("resourceFieldRef"), "", "resourceFieldRef is not supported"))
+	}
+	if ev.ValueFrom.ConfigMapKeyRef != nil {
+		numSources++
+		allErrs = append(allErrs, validateConfigMapKeySelector(ev.ValueFrom.ConfigMapKeyRef, fldPath.Child("configMapKeyRef"))...)
+	}
+	if ev.ValueFrom.SecretKeyRef != nil {
+		numSources++
+		allErrs = append(allErrs, validateSecretKeySelector(ev.ValueFrom.SecretKeyRef, fldPath.Child("secretKeyRef"))...)
+	}
+
+	if numSources == 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath, "", "must specify one of: `configMapKeyRef` or `secretKeyRef`"))
+	} else if len(ev.Value) != 0 {
+		if numSources != 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath, "", "may not be specified when `value` is not empty"))
+		}
+	} else if numSources > 1 {
+		allErrs = append(allErrs, field.Invalid(fldPath, "", "may not have more than one field specified at a time"))
+	}
+
+	return allErrs
+}
+
+func validateConfigMapKeySelector(s *corev1.ConfigMapKeySelector, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	for _, msg := range apivalidation.NameIsDNSSubdomain(s.Name, false) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), s.Name, msg))
+	}
+	if len(s.Key) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("key"), ""))
+	} else {
+		for _, msg := range validation.IsConfigMapKey(s.Key) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("key"), s.Key, msg))
+		}
+	}
+
+	return allErrs
+}
+
+func validateSecretKeySelector(s *corev1.SecretKeySelector, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	for _, msg := range apivalidation.NameIsDNSSubdomain(s.Name, false) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), s.Name, msg))
+	}
+	if len(s.Key) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("key"), ""))
+	} else {
+		for _, msg := range validation.IsConfigMapKey(s.Key) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("key"), s.Key, msg))
+		}
 	}
+
 	return allErrs
 }