Better helm charting (#236)

* WIP - introducing imagePullSecrets

* Adding test for imagepullsecrets

* now creates a docker secret

* the secret is created and passed to the helm chart

* refactoring helm install to always use the imagepullsecret

* refactor example cluster creation to honor the operator secret

* cluster wide install test now honors imagepullsecret

* upgrade operator test should honor ImagePullSecret

* import fix

* added write-up for the helm chart

* added test doc

* revert upgrade_operator test changes

README.md

@@ -189,6 +189,7 @@ deploymentName: cass-operator
 deploymentReplicas: 1
 image: "datastax/cass-operator:1.4.1"
 imagePullPolicy: IfNotPresent
+imagePullSecret: ""
 ```
 
 NOTE: roleName and roleBindingName will be used for a clusterRole and clusterRoleBinding if clusterWideInstall is set to true.
@@ -204,6 +205,36 @@ kubectl create namespace cass-operator-system
 helm install --set clusterWideInstall=true --namespace=cass-operator-system cass-operator ./charts/cass-operator-chart
 ```
 
+#### Using a custom Docker registry with the Helm Chart
+
+A custom Docker registry may be used as the source of the operator Docker image.  Before "helm install" is run, a Secret of type "docker-registry" should be created with the proper credentials.
+
+Then the "imagePullSecret" helm value may be set to the name of the ImagePullSecret to cause the custom Docker registry to be used.
+
+##### Custom Docker registry example: Github packages
+
+Github Packages may be used as a custom Docker registry.
+
+First, a Github personal access token must be created.
+
+See:
+
+https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token
+
+Second, the access token will be used to create the Secret:
+
+```console
+kubectl create secret docker-registry github-docker-registry --docker-username=USERNAME --docker-password=ACCESSTOKEN --docker-server docker.pkg.github.com
+```
+
+Replace USERNAME with the github username and ACCESSTOKEN with the personal access token.
+
+Now we can run "helm install" with the override value for imagePullSecret.  This is often used with an override value for image so that a specific tag can be chosen.  Note that the image value should include the full path to the custom registry.
+
+```console
+helm install --set image=docker.pkg.github.com/datastax/cass-operator/operator:latest-ubi --set imagePullSecrets=github-docker-registry cass-operator ./charts/cass-operator-chart
+```
+
 ## Features
 
 - Proper token ring initialization, with only one node bootstrapping at a time

charts/cass-operator-chart/templates/deployment.yaml

@@ -13,6 +13,10 @@ spec:
         name: cass-operator
     spec:
       serviceAccountName: {{ .Values.serviceAccountName }}
+    {{- if .Values.imagePullSecret }}
+      imagePullSecrets:
+      - name: {{ .Values.imagePullSecret }}
+    {{- end }}
       volumes:
       - name: tmpconfig-volume
         emptyDir:

charts/cass-operator-chart/values.yaml

@@ -11,3 +11,4 @@ deploymentName: cass-operator
 deploymentReplicas: 1
 image: "datastax/cass-operator:1.4.1"
 imagePullPolicy: IfNotPresent
+imagePullSecret: ""

mage/ginkgo/lib.go

@@ -22,7 +22,8 @@ import (
 )
 
 const (
-	EnvNoCleanup = "M_NO_CLEANUP"
+	EnvNoCleanup        = "M_NO_CLEANUP"
+	ImagePullSecretName = "imagepullsecret"
 )
 
 func duplicate(value string, count int) string {
@@ -370,18 +371,51 @@ func (ns *NsWrapper) WaitForOperatorReady() {
 	ns.WaitForOutputAndLog(step, k, "true", 240)
 }
 
+// kubectl create secret docker-registry github-docker-registry --docker-username=USER --docker-password=PASS --docker-server docker.pkg.github.com
+func CreateDockerRegistrySecret(name string, namespace string) {
+	args := []string{"secret", "docker-registry", name}
+	flags := map[string]string{
+		"docker-username": os.Getenv(kubectl.EnvDockerUsername),
+		"docker-password": os.Getenv(kubectl.EnvDockerPassword),
+		"docker-server":   os.Getenv(kubectl.EnvDockerServer),
+	}
+	k := kubectl.KCmd{Command: "create", Args: args, Flags: flags}
+	k.InNamespace(namespace).ExecVCapture()
+}
+
 func (ns NsWrapper) HelmInstall(chartPath string) {
-	var overrides = map[string]string{"image": cfgutil.GetOperatorImage()}
-	err := helm_util.Install(chartPath, "cass-operator", ns.Namespace, overrides)
-	mageutil.PanicOnError(err)
+	HelmInstallWithOverrides(chartPath, ns.Namespace, map[string]string{})
 }
 
 func (ns NsWrapper) HelmInstallWithPSPEnabled(chartPath string) {
 	var overrides = map[string]string{
-		"image":            cfgutil.GetOperatorImage(),
 		"vmwarePSPEnabled": "true",
 	}
-	err := helm_util.Install(chartPath, "cass-operator", ns.Namespace, overrides)
+	HelmInstallWithOverrides(chartPath, ns.Namespace, overrides)
+}
+
+// This is not a method on NsWrapper to allow mage to use it to create an example cluster.
+func HelmInstallWithOverrides(chartPath string, namespace string, overrides map[string]string) {
+	overrides["image"] = cfgutil.GetOperatorImage()
+
+	if kubectl.DockerCredentialsDefined() {
+		CreateDockerRegistrySecret(ImagePullSecretName, namespace)
+		overrides["imagePullSecret"] = ImagePullSecretName
+	}
+
+	err := helm_util.Install(chartPath, "cass-operator", namespace, overrides)
+	mageutil.PanicOnError(err)
+}
+
+func HelmUpgradeWithOverrides(chartPath string, namespace string, overrides map[string]string) {
+	overrides["image"] = cfgutil.GetOperatorImage()
+
+	// NOTE: This assumes the credential has already been defined during HelmInstallWithOverrides
+	if kubectl.DockerCredentialsDefined() {
+		overrides["imagePullSecret"] = ImagePullSecretName
+	}
+
+	err := helm_util.Upgrade(chartPath, "cass-operator", namespace, overrides)
 	mageutil.PanicOnError(err)
 }
 

mage/k8s/lib.go

@@ -12,7 +12,6 @@ import (
 	cfgutil "github.com/datastax/cass-operator/mage/config"
 	gcp "github.com/datastax/cass-operator/mage/gcloud"
 	ginkgo_util "github.com/datastax/cass-operator/mage/ginkgo"
-	helm_util "github.com/datastax/cass-operator/mage/helm"
 	integutil "github.com/datastax/cass-operator/mage/integ-tests"
 	k3d "github.com/datastax/cass-operator/mage/k3d"
 	kind "github.com/datastax/cass-operator/mage/kind"
@@ -145,10 +144,10 @@ func SetupExampleCluster() {
 	mg.Deps(SetupEmptyCluster)
 	kubectl.CreateSecretLiteral("cassandra-superuser-secret", "devuser", "devpass").ExecVPanic()
 
-	overrides := map[string]string{"image": getOperatorImage()}
 	var namespace = "default"
-	err := helm_util.Install("./charts/cass-operator-chart", "cass-operator", namespace, overrides)
-	mageutil.PanicOnError(err)
+	overrides := map[string]string{}
+
+	ginkgo_util.HelmInstallWithOverrides("./charts/cass-operator-chart", namespace, overrides)
 
 	// Wait for 15 seconds for the operator to come up
 	// because the apiserver will call the webhook too soon and fail if we do not wait

mage/kubectl/lib.go

@@ -16,6 +16,13 @@ import (
 	mageutil "github.com/datastax/cass-operator/mage/util"
 )
 
+const (
+	// Credentials for creating an ImagePullSecret
+	EnvDockerUsername = "M_DOCKER_USERNAME"
+	EnvDockerPassword = "M_DOCKER_PASSWORD"
+	EnvDockerServer   = "M_DOCKER_SERVER"
+)
+
 func GetKubeconfig(createDefault bool) string {
 	usr, err := user.Current()
 	if err != nil {
@@ -316,3 +323,8 @@ func GetNodeNameForPod(podName string) KCmd {
 	json := "jsonpath={.spec.nodeName}"
 	return Get(fmt.Sprintf("pod/%s", podName)).FormatOutput(json)
 }
+
+func DockerCredentialsDefined() bool {
+	_, ok := os.LookupEnv(EnvDockerUsername)
+	return ok
+}

tests/README.md

@@ -44,6 +44,25 @@ not automatically run the tests or tear down, run:
 M_K8S_FLAVOR=k3d mage k8s:setupEmptyCluster
 ```
 
+### Using a custom Docker registry for the operator image
+
+If M_DOCKER_USERNAME, M_DOCKER_PASSWORD, and M_DOCKER_SERVER environment variables are defined, then they will be used to create an image pull secret.  This secret will automatically be used by the integration tests for pulling the Docker image of the operator.
+
+This should be used in conjunction with the M_OPERATOR_IMAGE environment variable to select a specific image.  Note that the M_OPERATOR_IMAGE value should include the name of the custom registry.
+
+Example:
+
+```console
+export M_DOCKER_USERNAME=USERNAME
+export M_DOCKER_PASSWORD=ACCESSTOKEN
+export M_DOCKER_SERVER="docker.pkg.github.com"
+export M_OPERATOR_IMAGE="docker.pkg.github.com/datastax/cass-operator/operator:latest-ubi"
+```
+
+Replace USERNAME with the Github username and ACCESSTOKEN with a Github access token in the above commands.
+
+Note: The automatically created image pull secret will be named "imagepullsecret", and it will be removed at the end of each test when the test namespace is deleted.
+
 ### Kicking off the tests on an existing cluster
 To kick off all integration tests against the cluster that your kubectl
 is currently configured against, run:

tests/cluster_wide_install/cluster_wide_install_suite_test.go

@@ -58,15 +58,11 @@ var _ = Describe(testName, func() {
 			err := kubectl.CreateNamespace(opNamespace).ExecV()
 			Expect(err).ToNot(HaveOccurred())
 
-			//step := "setting up cass-operator resources via helm chart"
-
 			var overrides = map[string]string{
-				"image":              cfgutil.GetOperatorImage(),
 				"clusterWideInstall": "true",
 			}
 			chartPath := "../../charts/cass-operator-chart"
-			err = helm_util.Install(chartPath, "cass-operator", ns.Namespace, overrides)
-			mageutil.PanicOnError(err)
+			ginkgo_util.HelmInstallWithOverrides(chartPath, ns.Namespace, overrides)
 
 			ns.WaitForOperatorReady()
 

tests/helm_chart_imagepullsecrets/helm_chart_imagepullsecrets_suite_test.go

@@ -0,0 +1,65 @@
+// Copyright DataStax, Inc.
+// Please see the included license file for details.
+
+package helm_chart_imagepullsecrets
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	ginkgo_util "github.com/datastax/cass-operator/mage/ginkgo"
+	"github.com/datastax/cass-operator/mage/kubectl"
+)
+
+var (
+	testName    = "Helm Chart imagePullSecrets"
+	opNamespace = "test-helm-chart-imagepullsecrets"
+	dc1Name     = "dc2"
+	dc1Yaml     = "../testdata/default-single-rack-single-node-dc.yaml"
+	ns          = ginkgo_util.NewWrapper(testName, opNamespace)
+)
+
+func TestLifecycle(t *testing.T) {
+	// Only run this test if docker creds are defined
+	if kubectl.DockerCredentialsDefined() {
+		AfterSuite(func() {
+			logPath := fmt.Sprintf("%s/aftersuite", ns.LogDir)
+			err := kubectl.DumpAllLogs(logPath).ExecV()
+			if err != nil {
+				fmt.Printf("\n\tError during dumping logs: %s\n\n", err.Error())
+			}
+			fmt.Printf("\n\tPost-run logs dumped at: %s\n\n", logPath)
+			ns.Terminate()
+		})
+
+		RegisterFailHandler(Fail)
+		RunSpecs(t, testName)
+	}
+}
+
+var _ = Describe(testName, func() {
+	Context("when in a new cluster", func() {
+		Specify("the operator can be correctly installed with imagePullSecrets", func() {
+
+			By("creating a namespace for the cass-operator")
+			err := kubectl.CreateNamespace(opNamespace).ExecV()
+			Expect(err).ToNot(HaveOccurred())
+
+			step := "setting up cass-operator resources via helm chart"
+
+			ns.HelmInstall("../../charts/cass-operator-chart")
+
+			ns.WaitForOperatorReady()
+
+			// Create a small cass cluster to verify cass-operator is functional
+			step = "creating first datacenter resource"
+			k := kubectl.ApplyFiles(dc1Yaml)
+			ns.ExecAndLog(step, k)
+
+			ns.WaitForDatacenterReady(dc1Name)
+		})
+	})
+})