Use the controller ServiceAccount for tests

Previously the tests ran the controller binary locally using the default kubeadmin kubeconfig. This generates a new kubeconfig using the controller ServiceAccount for the local binary to use. ref: https://issues.redhat.com/browse/ACM-2439 Signed-off-by: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com>
open-cluster-management-io · Feb 7, 2023 · 1f9464f · 1f9464f
1 parent 81a1487
commit 1f9464f
Show file tree

Hide file tree

Showing 8 changed files with 87 additions and 48 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -67,10 +67,14 @@ jobs:
  run: |
  make kind-bootstrap-cluster-dev
 
+ - name: Ensure Service Account kubeconfig
+ run: |
+ KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
+
  - name: E2E Tests
  run: |
  export GOPATH=$(go env GOPATH)
- make e2e-test-coverage
+ KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
 
  - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
  env:
@@ -81,7 +85,7 @@ jobs:
  - name: E2E tests that simulate hosted mode
  run: |
  export GOPATH=$(go env GOPATH)
- make e2e-test-hosted-mode-coverage
+ KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
 
  - name: Test Coverage Verification
  if: ${{ github.event_name == 'pull_request' }}
@@ -92,7 +96,7 @@ jobs:
  - name: Verify Deployment Configuration
  run: |
  make build-images
- make kind-deploy-controller-dev
+ KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
 
  - name: Debug
  if: ${{ failure() }}

diff --git a/.gitignore b/.gitignore
@@ -87,8 +87,7 @@ test/crds/test-policy.yaml
 !vbh/.build-harness-vendorized
 !vbh/build-harness
 !vbh/build-harness-extensions 
-kubeconfig_managed
-kubeconfig_managed2
+kubeconfig_managed*
 kind_kubeconfig.yaml
 report.json
 report_*.json

diff --git a/Makefile b/Makefile
@@ -28,12 +28,15 @@ TESTARGS_DEFAULT := -v
 export TESTARGS ?= $(TESTARGS_DEFAULT)
 VERSION ?= $(shell cat COMPONENT_VERSION 2> /dev/null)
 IMAGE_NAME_AND_VERSION ?= $(REGISTRY)/$(IMG)
+CONTROLLER_NAMESPACE ?= open-cluster-management-agent-addon
 # Handle KinD configuration
-KIND_NAME ?= test-managed
-KIND_NAMESPACE ?= open-cluster-management-agent-addon
-KIND_VERSION ?= latest
-MANAGED_CLUSTER_NAME ?= managed
+MANAGED_CLUSTER_SUFFIX ?= 
+MANAGED_CLUSTER_NAME ?= managed$(MANAGED_CLUSTER_SUFFIX)
 WATCH_NAMESPACE ?= $(MANAGED_CLUSTER_NAME)
+KIND_NAME ?= test-$(MANAGED_CLUSTER_NAME)
+KIND_CLUSTER_NAME ?= kind-$(KIND_NAME)
+KIND_NAMESPACE ?= $(CONTROLLER_NAMESPACE)
+KIND_VERSION ?= latest
 # Set the Kind version tag
 ifeq ($(KIND_VERSION), minimum)
  KIND_ARGS = --image kindest/node:v1.19.16
@@ -52,6 +55,13 @@ IMG ?= $(shell cat COMPONENT_NAME 2> /dev/null)
 REGISTRY ?= quay.io/open-cluster-management
 TAG ?= latest
 
+# Handle base64 OS differences
+OS = $(shell uname -s | tr '[:upper:]' '[:lower:]')
+BASE64 = base64 -w 0
+ifeq ($(OS), darwin)
+ BASE64 = base64
+endif
+
 # go-get-tool will 'go install' any package $1 and install it to LOCAL_BIN.
 define go-get-tool
 @set -e ;\
@@ -195,7 +205,7 @@ clean:
  -rm build/_output/bin/*
  -rm coverage*.out
  -rm report*.json
- -rm kubeconfig_managed
+ -rm kubeconfig_managed*
  -rm -r vendor/
 
 ############################################################
@@ -235,17 +245,13 @@ kustomize: ## Download kustomize locally if necessary.
 GINKGO = $(LOCAL_BIN)/ginkgo
 
 .PHONY: kind-bootstrap-cluster
-kind-bootstrap-cluster: kind-create-cluster install-crds kind-deploy-controller install-resources
+kind-bootstrap-cluster: kind-bootstrap-cluster-dev kind-deploy-controller
 
 .PHONY: kind-bootstrap-cluster-dev
-kind-bootstrap-cluster-dev: kind-create-cluster install-crds install-resources
+kind-bootstrap-cluster-dev: kind-create-cluster install-crds kind-controller-kubeconfig
 
 .PHONY: kind-deploy-controller
-kind-deploy-controller: generate-operator-yaml
- @echo Installing $(IMG)
- -kubectl create ns $(KIND_NAMESPACE)
- kubectl apply -f deploy/operator.yaml -n $(KIND_NAMESPACE)
- kubectl patch deployment $(IMG) -n $(KIND_NAMESPACE) -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$(IMG)\",\"env\":[{\"name\":\"WATCH_NAMESPACE\",\"value\":\"$(WATCH_NAMESPACE)\"}]}]}}}}"
+kind-deploy-controller: generate-operator-yaml install-resources deploy
 
 .PHONY: deploy-controller
 deploy-controller: kind-deploy-controller
@@ -262,16 +268,27 @@ kind-deploy-controller-dev: kind-deploy-controller
 # Specify KIND_VERSION to indicate the version tag of the KinD image
 .PHONY: kind-create-cluster
 kind-create-cluster:
- @echo "creating cluster"
- kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
- kind get kubeconfig --name $(KIND_NAME) > $(PWD)/kubeconfig_managed
+ # ensuring cluster $(KIND_NAME)
+ -kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
+ kubectl config use-context $(KIND_CLUSTER_NAME)
+ kind get kubeconfig --name $(KIND_NAME) > kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e
+
+.PHONY: kind-controller-kubeconfig
+kind-controller-kubeconfig: install-resources
+ kubectl -n $(KIND_NAMESPACE) apply -f test/resources/e2e_controller_secret.yaml
+ -rm kubeconfig_$(MANAGED_CLUSTER_NAME)
+ @kubectl config set-cluster $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
+ --server=$(shell kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}' --kubeconfig=kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e) \
+ --insecure-skip-tls-verify=true
+ @kubectl config set-credentials $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
+ --token=$$(kubectl get secret -n $(KIND_NAMESPACE) config-policy-controller -o jsonpath='{.data.token}' --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e | $(BASE64) --decode)
+ @kubectl config set-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
+ --user=$(KIND_CLUSTER_NAME) --cluster=$(KIND_CLUSTER_NAME)
+ @kubectl config use-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME)
 
 .PHONY: kind-additional-cluster
-kind-additional-cluster:
- @echo "creating cluster"
- kind create cluster --name $(KIND_NAME)2 $(KIND_ARGS)
- kind get kubeconfig --name $(KIND_NAME)2 > $(PWD)/kubeconfig_managed2
- kubectl config use-context kind-$(KIND_NAME)
+kind-additional-cluster: MANAGED_CLUSTER_SUFFIX = 2
+kind-additional-cluster: kind-create-cluster kind-controller-kubeconfig
 
 .PHONY: kind-delete-cluster
 kind-delete-cluster:
@@ -294,8 +311,19 @@ install-crds:
 
 .PHONY: install-resources
 install-resources:
- @echo creating namespaces
- kubectl create ns $(WATCH_NAMESPACE)
+ # creating namespaces
+ -kubectl create ns $(WATCH_NAMESPACE)
+ -kubectl create ns $(KIND_NAMESPACE)
+ # deploying roles and service account
+ kubectl apply -k deploy/rbac
+ kubectl apply -f deploy/manager/service-account.yaml -n $(KIND_NAMESPACE)
+
+.PHONY: kind-ensure-sa
+kind-ensure-sa:
+ @KUBECONFIG_TOKEN="$$(kubectl config view --raw -o jsonpath='{.users[].user.token}')"; \
+ KUBECONFIG_USER="$$(echo "$${KUBECONFIG_TOKEN}" | jq -rR 'split(".") | .[1] | select(. != null) | @base64d | fromjson | .sub')"; \
+ echo "Kubeconfig user detected from token: $${KUBECONFIG_USER}"; \
+ [ "$${KUBECONFIG_USER}" = "system:serviceaccount:$(KIND_NAMESPACE):config-policy-controller" ]
 
 .PHONY: e2e-dependencies
 e2e-dependencies:
@@ -306,7 +334,7 @@ e2e-test: e2e-dependencies
  $(GINKGO) -v --fail-fast --slow-spec-threshold=10s $(E2E_TEST_ARGS) test/e2e
 
 .PHONY: e2e-test-coverage
-e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter="!hosted-mode" --output-dir=.
+e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode' --output-dir=.
 e2e-test-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented
 
 .PHONY: e2e-test-hosted-mode-coverage

diff --git a/deploy/manager/kustomization.yaml b/deploy/manager/kustomization.yaml
@@ -1,3 +1,4 @@
 resources:
 - ./manager.yaml
+- ./service-account.yaml
 - ../rbac/
diff --git a/deploy/manager/manager.yaml b/deploy/manager/manager.yaml
@@ -33,21 +33,3 @@ spec:
  fieldPath: metadata.name
  - name: OPERATOR_NAME
  value: "config-policy-controller"
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: config-policy-controller
-subjects:
-- kind: ServiceAccount
- name: config-policy-controller
- namespace: open-cluster-management-agent-addon
-roleRef:
- kind: ClusterRole
- name: config-policy-controller
- apiGroup: rbac.authorization.k8s.io
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: config-policy-controller
diff --git a/deploy/manager/service-account.yaml b/deploy/manager/service-account.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: config-policy-controller
+subjects:
+- kind: ServiceAccount
+ name: config-policy-controller
+ namespace: open-cluster-management-agent-addon
+roleRef:
+ kind: ClusterRole
+ name: config-policy-controller
+ apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: config-policy-controller
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -56,8 +56,8 @@ func TestE2e(t *testing.T) {
 func init() {
  klog.SetOutput(GinkgoWriter)
  klog.InitFlags(nil)
- flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed",
- "Location of the kubeconfig to use; defaults to KUBECONFIG if not set")
+ flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed_e2e",
+ "Location of the kubeconfig to use; defaults to current kubeconfig if set to an empty string")
 }
 
 var _ = BeforeSuite(func() {

diff --git a/test/resources/e2e_controller_secret.yaml b/test/resources/e2e_controller_secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+ name: config-policy-controller
+ annotations:
+ kubernetes.io/service-account.name: config-policy-controller