CDAP-19300 adding container injection feature and Dockerfile for unit testing on Docker image

.gitignore

@@ -63,6 +63,7 @@ anaconda-mode/
 *.dylib
 # Test binary, build with 'go test -c'
 *.test
+!Dockerfile.test
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 ### Vim ###

Dockerfile.test

@@ -0,0 +1,27 @@
+# Build the manager binary
+FROM golang:1.19 as tester
+
+ENV version 1.0.8
+ENV arch amd64
+
+# Copy everything in the go src
+WORKDIR /go/src/cdap.io/cdap-operator
+COPY ./ ./
+
+# Install Kubebuilder
+RUN curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${version}/kubebuilder_${version}_linux_${arch}.tar.gz" && \
+    tar -zxvf kubebuilder_${version}_linux_${arch}.tar.gz && \
+    mv kubebuilder_${version}_linux_${arch} /usr/local/kubebuilder && \
+    cp /usr/local/kubebuilder/bin/* /usr/local/bin
+
+# Install setup-envtest
+RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+RUN go env -w GO111MODULE=auto
+
+# download envtest 1.19.x for kubebuilder and to set KUBEBUILDER_ASSETS environment variable
+RUN $(go env GOPATH)/bin/setup-envtest use -p env 1.19.x > /tmp/setup_envtest.sh && \
+    eval `$(go env GOPATH)/bin/setup-envtest use -p env 1.19.x` && \
+    rm /tmp/setup_envtest.sh
+
+CMD make test

Makefile

@@ -3,6 +3,7 @@
 IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.22
+CRD_OPTIONS ?= "crd:maxDescLen=0"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -41,7 +42,7 @@ help: ## Display this help.
 
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.

README.md

@@ -82,3 +82,10 @@ rm /tmp/setup_envtest.sh
 ```
 
 4. Run `make test`
+
+#### Running Unit Tests in a Docker image
+
+From the project root folder build, execute and then remove the test image by running the following
+```
+docker build -f Dockerfile.test . -t test && docker run --rm -it test; docker image rm test
+```

api/v1alpha1/cdapmaster_types.go

@@ -168,6 +168,10 @@ type CDAPServiceSpec struct {
 	Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"`
 	// Affinity describes node affinity scheduling rules for the service.
 	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+	// Containers define any additional containers a service has
+	// This is a list of containers and can be left blank
+	// A typical use is to add sidecars for a deployment
+	Containers []*corev1.Container `json:"containers,omitempty"`
 }
 
 // CDAPScalableServiceSpec defines the base specification for master services that can have more than one instance.

controllers/cdapmaster_controller.go

@@ -128,9 +128,41 @@ func ApplyDefaults(resource interface{}) {
 	}
 
 	// Set the configMapCConf entry for the router and UI service and ports
-	spec.Config[confRouterServerAddress] = fmt.Sprintf("cdap-%s-%s", r.Name, strings.ToLower(string(serviceRouter)))
-	spec.Config[confRouterBindPort] = strconv.Itoa(int(*spec.Router.ServicePort))
-	spec.Config[confUserInterfaceBindPort] = strconv.Itoa(int(*spec.UserInterface.ServicePort))
+	if spec.Config[confRouterServerAddress] == "" {
+		spec.Config[confRouterServerAddress] = fmt.Sprintf("cdap-%s-%s", r.Name, strings.ToLower(string(serviceRouter)))
+	}
+	if spec.Config[confRouterBindPort] == "" {
+		spec.Config[confRouterBindPort] = strconv.Itoa(int(*spec.Router.ServicePort))
+	}
+	if spec.Config[confUserInterfaceBindPort] == "" {
+		spec.Config[confUserInterfaceBindPort] = strconv.Itoa(int(*spec.UserInterface.ServicePort))
+	}
+
+	// Set the default local data directory if it is not set in cdap-cr.
+	if _, ok := spec.Config[confLocalDataDirKey]; !ok {
+		spec.Config[confLocalDataDirKey] = confLocalDataDirVal
+	}
+
+	// Set security secret disk names to be consistent with securitySecret if not overwritten.
+	if _, ok := spec.Config[confTwillSecurityMasterSecretDiskName]; !ok && spec.SecuritySecret != "" {
+		spec.Config[confTwillSecurityMasterSecretDiskName] = spec.SecuritySecret
+	}
+	if _, ok := spec.Config[confTwillSecurityMasterSecretDiskPath]; !ok && spec.SecuritySecret != "" {
+		spec.Config[confTwillSecurityMasterSecretDiskPath] = defaultSecuritySecretPath
+	}
+	// This configuration makes the default securitySecret available to the workers by default.
+	// TODO: Add support for secure-by-default configurations.
+	if _, ok := spec.Config[confTwillSecurityWorkerSecretDiskName]; !ok && spec.SecuritySecret != "" {
+		spec.Config[confTwillSecurityWorkerSecretDiskName] = spec.SecuritySecret
+	}
+	if _, ok := spec.Config[confTwillSecurityWorkerSecretDiskPath]; !ok && spec.SecuritySecret != "" {
+		spec.Config[confTwillSecurityWorkerSecretDiskPath] = defaultSecuritySecretPath
+	}
+
+	// Set the default JMX server port if not set and system metrics exporter sidecar is enabled
+	if _, ok := spec.Config[confJMXServerPort]; spec.SystemMetricsExporter != nil && !ok {
+		spec.Config[confJMXServerPort] = fmt.Sprint(defaultJMXport)
+	}
 
 	// Set the default local data directory if it is not set in cdap-cr.
 	if _, ok := spec.Config[confLocalDataDirKey]; !ok {
@@ -166,9 +198,9 @@ func ApplyDefaults(resource interface{}) {
 	finalizer.EnsureStandard(r)
 }
 
-/////////////////////////////////////////////////////////
-///// Handling reconciling ConfigMapHandler objects /////
-/////////////////////////////////////////////////////////
+// ///////////////////////////////////////////////////////
+// /// Handling reconciling ConfigMapHandler objects /////
+// ///////////////////////////////////////////////////////
 type ConfigMapHandler struct{}
 
 func (h *ConfigMapHandler) Observables(rsrc interface{}, labels map[string]string, dependent []reconciler.Object) []reconciler.Observable {
@@ -257,9 +289,9 @@ func buildConfigMapObject(spec *ConfigMapSpec) reconciler.Object {
 	return obj
 }
 
-///////////////////////////////////////////////////////////
-///// Handling reconciling deployment of all services /////
-///////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////
+// /// Handling reconciling deployment of all services /////
+// /////////////////////////////////////////////////////////
 type ServiceHandler struct{}
 
 func (h *ServiceHandler) Observables(rsrc interface{}, labels map[string]string, dependent []reconciler.Object) []reconciler.Observable {
@@ -337,9 +369,9 @@ func CopyNodePortIfAny(expected, observed []reconciler.Object) {
 	}
 }
 
-///////////////////////////////////////////////////////
-///// Handler for image version upgrade/downgrade /////
-///////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////
+// /// Handler for image version upgrade/downgrade /////
+// /////////////////////////////////////////////////////
 type VersionUpdateHandler struct{}
 
 func (h *VersionUpdateHandler) Observables(rsrc interface{}, labels map[string]string, dependent []reconciler.Object) []reconciler.Observable {

controllers/deployment.go

@@ -202,6 +202,14 @@ func buildStatefulSets(master *v1alpha1.CDAPMaster, name string, services Servic
 			return nil, err
 		}
 		spec = spec.withContainer(c)
+
+		if ss.Containers != nil {
+			for _, container := range ss.Containers {
+				additionalContainer := containerSpecFromContainer(container, dataDir)
+				spec = spec.withContainer(additionalContainer)
+			}
+		}
+
 		if err := addSystemMetricsServiceIfEnabled(spec, master, ss, dataDir, c); err != nil {
 			return nil, err
 		}
@@ -344,6 +352,13 @@ func buildDeployment(master *v1alpha1.CDAPMaster, name string, services ServiceG
 		}
 		spec = spec.withContainer(c)
 
+		if ss.Containers != nil {
+			for _, container := range ss.Containers {
+				additionalContainer := containerSpecFromContainer(container, dataDir)
+				spec = spec.withContainer(additionalContainer)
+			}
+		}
+
 		// Adding a label to allow k8s service selector to easily find the pod
 		spec = spec.addLabel(labelContainerKeyPrefix+s, master.Name)
 
@@ -477,10 +492,15 @@ func buildStatefulSetsObject(spec *StatefulSpec) (*reconciler.Object, error) {
 		}
 	}
 	for index, _ := range statefulSetObj.Spec.Template.Spec.Containers {
-		if err := addVolumeMountToContainer(&statefulSetObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+		container := &statefulSetObj.Spec.Template.Spec.Containers[index]
+		if err := addVolumeMountToContainer(container, spec.Base.AdditionalVolumeMounts); err != nil {
 			return nil, err
 		}
-		setLifecycleHookForContainer(&statefulSetObj.Spec.Template.Spec.Containers[index], spec.Containers[index].Lifecycle)
+		setEnvForContainer(container, spec.Containers[index].Env)
+		setLifecycleHookForContainer(container, spec.Containers[index].Lifecycle)
+		setLivenessProbeForContainer(container, spec.Containers[index].LivenessProbe)
+		setReadinessProbeForContainer(container, spec.Containers[index].ReadinessProbe)
+		setPortsForContainer(container, spec.Containers[index].Ports)
 	}
 	return obj, nil
 }
@@ -511,10 +531,15 @@ func buildDeploymentObject(spec *DeploymentSpec) (*reconciler.Object, error) {
 		}
 	}
 	for index, _ := range deploymentObj.Spec.Template.Spec.Containers {
-		if err := addVolumeMountToContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+		container := &deploymentObj.Spec.Template.Spec.Containers[index]
+		if err := addVolumeMountToContainer(container, spec.Base.AdditionalVolumeMounts); err != nil {
 			return nil, err
 		}
-		setLifecycleHookForContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Containers[index].Lifecycle)
+		setEnvForContainer(container, spec.Containers[index].Env)
+		setLifecycleHookForContainer(container, spec.Containers[index].Lifecycle)
+		setLivenessProbeForContainer(container, spec.Containers[index].LivenessProbe)
+		setReadinessProbeForContainer(container, spec.Containers[index].ReadinessProbe)
+		setPortsForContainer(container, spec.Containers[index].Ports)
 	}
 	return obj, nil
 }
@@ -543,10 +568,26 @@ func addVolumeMountToContainer(container *corev1.Container, volumeMountsToAdd []
 	return nil
 }
 
+func setEnvForContainer(container *corev1.Container, env []corev1.EnvVar) {
+	container.Env = env
+}
+
 func setLifecycleHookForContainer(container *corev1.Container, lifecycle *corev1.Lifecycle) {
 	container.Lifecycle = lifecycle
 }
 
+func setLivenessProbeForContainer(container *corev1.Container, livenessProbe *corev1.Probe) {
+	container.LivenessProbe = livenessProbe
+}
+
+func setReadinessProbeForContainer(container *corev1.Container, readinessProbe *corev1.Probe) {
+	container.ReadinessProbe = readinessProbe
+}
+
+func setPortsForContainer(container *corev1.Container, ports []corev1.ContainerPort) {
+	container.Ports = ports
+}
+
 // Return a NodePort service to expose the supplied target service
 func buildNetworkService(master *v1alpha1.CDAPMaster, name NetworkServiceName, target ServiceName, labels map[string]string) (*NetworkServiceSpec, error) {
 	s, err := getCDAPExternalServiceSpec(master, target)