From 1f9464f6b9ad6deeefcfe6206c537f8b10c3c8a0 Mon Sep 17 00:00:00 2001 From: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:34:44 -0500 Subject: [PATCH] Use the controller ServiceAccount for tests Previously the tests ran the controller binary locally using the default kubeadmin kubeconfig. This generates a new kubeconfig using the controller ServiceAccount for the local binary to use. ref: https://issues.redhat.com/browse/ACM-2439 Signed-off-by: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com> --- .github/workflows/kind.yml | 10 ++- .gitignore | 3 +- Makefile | 74 ++++++++++++++++------- deploy/manager/kustomization.yaml | 1 + deploy/manager/manager.yaml | 18 ------ deploy/manager/service-account.yaml | 18 ++++++ test/e2e/e2e_suite_test.go | 4 +- test/resources/e2e_controller_secret.yaml | 7 +++ 8 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 deploy/manager/service-account.yaml create mode 100644 test/resources/e2e_controller_secret.yaml diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 9d1d5259..7ad4e0de 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -67,10 +67,14 @@ jobs: run: | make kind-bootstrap-cluster-dev + - name: Ensure Service Account kubeconfig + run: | + KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa + - name: E2E Tests run: | export GOPATH=$(go env GOPATH) - make e2e-test-coverage + KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }} env: @@ -81,7 +85,7 @@ jobs: - name: E2E tests that simulate hosted mode run: | export GOPATH=$(go env GOPATH) - make e2e-test-hosted-mode-coverage + KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage - name: Test Coverage Verification if: ${{ github.event_name == 'pull_request' }} @@ -92,7 +96,7 @@ jobs: - name: Verify Deployment Configuration run: | make build-images - make kind-deploy-controller-dev + KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev - name: Debug if: ${{ failure() }} diff --git a/.gitignore b/.gitignore index e3332d83..e56d9cb5 100644 --- a/.gitignore +++ b/.gitignore @@ -87,8 +87,7 @@ test/crds/test-policy.yaml !vbh/.build-harness-vendorized !vbh/build-harness !vbh/build-harness-extensions -kubeconfig_managed -kubeconfig_managed2 +kubeconfig_managed* kind_kubeconfig.yaml report.json report_*.json diff --git a/Makefile b/Makefile index 01cd9e6f..64c3c1f8 100644 --- a/Makefile +++ b/Makefile @@ -28,12 +28,15 @@ TESTARGS_DEFAULT := -v export TESTARGS ?= $(TESTARGS_DEFAULT) VERSION ?= $(shell cat COMPONENT_VERSION 2> /dev/null) IMAGE_NAME_AND_VERSION ?= $(REGISTRY)/$(IMG) +CONTROLLER_NAMESPACE ?= open-cluster-management-agent-addon # Handle KinD configuration -KIND_NAME ?= test-managed -KIND_NAMESPACE ?= open-cluster-management-agent-addon -KIND_VERSION ?= latest -MANAGED_CLUSTER_NAME ?= managed +MANAGED_CLUSTER_SUFFIX ?= +MANAGED_CLUSTER_NAME ?= managed$(MANAGED_CLUSTER_SUFFIX) WATCH_NAMESPACE ?= $(MANAGED_CLUSTER_NAME) +KIND_NAME ?= test-$(MANAGED_CLUSTER_NAME) +KIND_CLUSTER_NAME ?= kind-$(KIND_NAME) +KIND_NAMESPACE ?= $(CONTROLLER_NAMESPACE) +KIND_VERSION ?= latest # Set the Kind version tag ifeq ($(KIND_VERSION), minimum) KIND_ARGS = --image kindest/node:v1.19.16 @@ -52,6 +55,13 @@ IMG ?= $(shell cat COMPONENT_NAME 2> /dev/null) REGISTRY ?= quay.io/open-cluster-management TAG ?= latest +# Handle base64 OS differences +OS = $(shell uname -s | tr '[:upper:]' '[:lower:]') +BASE64 = base64 -w 0 +ifeq ($(OS), darwin) + BASE64 = base64 +endif + # go-get-tool will 'go install' any package $1 and install it to LOCAL_BIN. define go-get-tool @set -e ;\ @@ -195,7 +205,7 @@ clean: -rm build/_output/bin/* -rm coverage*.out -rm report*.json - -rm kubeconfig_managed + -rm kubeconfig_managed* -rm -r vendor/ ############################################################ @@ -235,17 +245,13 @@ kustomize: ## Download kustomize locally if necessary. GINKGO = $(LOCAL_BIN)/ginkgo .PHONY: kind-bootstrap-cluster -kind-bootstrap-cluster: kind-create-cluster install-crds kind-deploy-controller install-resources +kind-bootstrap-cluster: kind-bootstrap-cluster-dev kind-deploy-controller .PHONY: kind-bootstrap-cluster-dev -kind-bootstrap-cluster-dev: kind-create-cluster install-crds install-resources +kind-bootstrap-cluster-dev: kind-create-cluster install-crds kind-controller-kubeconfig .PHONY: kind-deploy-controller -kind-deploy-controller: generate-operator-yaml - @echo Installing $(IMG) - -kubectl create ns $(KIND_NAMESPACE) - kubectl apply -f deploy/operator.yaml -n $(KIND_NAMESPACE) - kubectl patch deployment $(IMG) -n $(KIND_NAMESPACE) -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$(IMG)\",\"env\":[{\"name\":\"WATCH_NAMESPACE\",\"value\":\"$(WATCH_NAMESPACE)\"}]}]}}}}" +kind-deploy-controller: generate-operator-yaml install-resources deploy .PHONY: deploy-controller deploy-controller: kind-deploy-controller @@ -262,16 +268,27 @@ kind-deploy-controller-dev: kind-deploy-controller # Specify KIND_VERSION to indicate the version tag of the KinD image .PHONY: kind-create-cluster kind-create-cluster: - @echo "creating cluster" - kind create cluster --name $(KIND_NAME) $(KIND_ARGS) - kind get kubeconfig --name $(KIND_NAME) > $(PWD)/kubeconfig_managed + # ensuring cluster $(KIND_NAME) + -kind create cluster --name $(KIND_NAME) $(KIND_ARGS) + kubectl config use-context $(KIND_CLUSTER_NAME) + kind get kubeconfig --name $(KIND_NAME) > kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e + +.PHONY: kind-controller-kubeconfig +kind-controller-kubeconfig: install-resources + kubectl -n $(KIND_NAMESPACE) apply -f test/resources/e2e_controller_secret.yaml + -rm kubeconfig_$(MANAGED_CLUSTER_NAME) + @kubectl config set-cluster $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \ + --server=$(shell kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}' --kubeconfig=kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e) \ + --insecure-skip-tls-verify=true + @kubectl config set-credentials $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \ + --token=$$(kubectl get secret -n $(KIND_NAMESPACE) config-policy-controller -o jsonpath='{.data.token}' --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e | $(BASE64) --decode) + @kubectl config set-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \ + --user=$(KIND_CLUSTER_NAME) --cluster=$(KIND_CLUSTER_NAME) + @kubectl config use-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) .PHONY: kind-additional-cluster -kind-additional-cluster: - @echo "creating cluster" - kind create cluster --name $(KIND_NAME)2 $(KIND_ARGS) - kind get kubeconfig --name $(KIND_NAME)2 > $(PWD)/kubeconfig_managed2 - kubectl config use-context kind-$(KIND_NAME) +kind-additional-cluster: MANAGED_CLUSTER_SUFFIX = 2 +kind-additional-cluster: kind-create-cluster kind-controller-kubeconfig .PHONY: kind-delete-cluster kind-delete-cluster: @@ -294,8 +311,19 @@ install-crds: .PHONY: install-resources install-resources: - @echo creating namespaces - kubectl create ns $(WATCH_NAMESPACE) + # creating namespaces + -kubectl create ns $(WATCH_NAMESPACE) + -kubectl create ns $(KIND_NAMESPACE) + # deploying roles and service account + kubectl apply -k deploy/rbac + kubectl apply -f deploy/manager/service-account.yaml -n $(KIND_NAMESPACE) + +.PHONY: kind-ensure-sa +kind-ensure-sa: + @KUBECONFIG_TOKEN="$$(kubectl config view --raw -o jsonpath='{.users[].user.token}')"; \ + KUBECONFIG_USER="$$(echo "$${KUBECONFIG_TOKEN}" | jq -rR 'split(".") | .[1] | select(. != null) | @base64d | fromjson | .sub')"; \ + echo "Kubeconfig user detected from token: $${KUBECONFIG_USER}"; \ + [ "$${KUBECONFIG_USER}" = "system:serviceaccount:$(KIND_NAMESPACE):config-policy-controller" ] .PHONY: e2e-dependencies e2e-dependencies: @@ -306,7 +334,7 @@ e2e-test: e2e-dependencies $(GINKGO) -v --fail-fast --slow-spec-threshold=10s $(E2E_TEST_ARGS) test/e2e .PHONY: e2e-test-coverage -e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter="!hosted-mode" --output-dir=. +e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode' --output-dir=. e2e-test-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented .PHONY: e2e-test-hosted-mode-coverage diff --git a/deploy/manager/kustomization.yaml b/deploy/manager/kustomization.yaml index 6abb0cb3..f0fcc74d 100644 --- a/deploy/manager/kustomization.yaml +++ b/deploy/manager/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ./manager.yaml +- ./service-account.yaml - ../rbac/ diff --git a/deploy/manager/manager.yaml b/deploy/manager/manager.yaml index 9ab7dc32..fe675cff 100644 --- a/deploy/manager/manager.yaml +++ b/deploy/manager/manager.yaml @@ -33,21 +33,3 @@ spec: fieldPath: metadata.name - name: OPERATOR_NAME value: "config-policy-controller" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: config-policy-controller -subjects: -- kind: ServiceAccount - name: config-policy-controller - namespace: open-cluster-management-agent-addon -roleRef: - kind: ClusterRole - name: config-policy-controller - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: config-policy-controller diff --git a/deploy/manager/service-account.yaml b/deploy/manager/service-account.yaml new file mode 100644 index 00000000..6ffe8ade --- /dev/null +++ b/deploy/manager/service-account.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: config-policy-controller +subjects: +- kind: ServiceAccount + name: config-policy-controller + namespace: open-cluster-management-agent-addon +roleRef: + kind: ClusterRole + name: config-policy-controller + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: config-policy-controller diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f952545e..29728457 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -56,8 +56,8 @@ func TestE2e(t *testing.T) { func init() { klog.SetOutput(GinkgoWriter) klog.InitFlags(nil) - flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed", - "Location of the kubeconfig to use; defaults to KUBECONFIG if not set") + flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed_e2e", + "Location of the kubeconfig to use; defaults to current kubeconfig if set to an empty string") } var _ = BeforeSuite(func() { diff --git a/test/resources/e2e_controller_secret.yaml b/test/resources/e2e_controller_secret.yaml new file mode 100644 index 00000000..7f7b43c7 --- /dev/null +++ b/test/resources/e2e_controller_secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: config-policy-controller + annotations: + kubernetes.io/service-account.name: config-policy-controller