Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the controller ServiceAccount for tests #101

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/workflows/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,14 @@ jobs:
run: |
make kind-bootstrap-cluster-dev
- name: Ensure Service Account kubeconfig
run: |
KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
- name: E2E Tests
run: |
export GOPATH=$(go env GOPATH)
make e2e-test-coverage
KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
- name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
env:
Expand All @@ -81,7 +85,7 @@ jobs:
- name: E2E tests that simulate hosted mode
run: |
export GOPATH=$(go env GOPATH)
make e2e-test-hosted-mode-coverage
KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
- name: Test Coverage Verification
if: ${{ github.event_name == 'pull_request' }}
Expand All @@ -92,7 +96,7 @@ jobs:
- name: Verify Deployment Configuration
run: |
make build-images
make kind-deploy-controller-dev
KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
- name: Debug
if: ${{ failure() }}
Expand Down
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ test/crds/test-policy.yaml
!vbh/.build-harness-vendorized
!vbh/build-harness
!vbh/build-harness-extensions
kubeconfig_managed
kubeconfig_managed2
kubeconfig_managed*
kind_kubeconfig.yaml
report.json
report_*.json
Expand Down
74 changes: 51 additions & 23 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,15 @@ TESTARGS_DEFAULT := -v
export TESTARGS ?= $(TESTARGS_DEFAULT)
VERSION ?= $(shell cat COMPONENT_VERSION 2> /dev/null)
IMAGE_NAME_AND_VERSION ?= $(REGISTRY)/$(IMG)
CONTROLLER_NAMESPACE ?= open-cluster-management-agent-addon
# Handle KinD configuration
KIND_NAME ?= test-managed
KIND_NAMESPACE ?= open-cluster-management-agent-addon
KIND_VERSION ?= latest
MANAGED_CLUSTER_NAME ?= managed
MANAGED_CLUSTER_SUFFIX ?=
MANAGED_CLUSTER_NAME ?= managed$(MANAGED_CLUSTER_SUFFIX)
WATCH_NAMESPACE ?= $(MANAGED_CLUSTER_NAME)
KIND_NAME ?= test-$(MANAGED_CLUSTER_NAME)
KIND_CLUSTER_NAME ?= kind-$(KIND_NAME)
KIND_NAMESPACE ?= $(CONTROLLER_NAMESPACE)
KIND_VERSION ?= latest
# Set the Kind version tag
ifeq ($(KIND_VERSION), minimum)
KIND_ARGS = --image kindest/node:v1.19.16
Expand All @@ -52,6 +55,13 @@ IMG ?= $(shell cat COMPONENT_NAME 2> /dev/null)
REGISTRY ?= quay.io/open-cluster-management
TAG ?= latest

# Handle base64 OS differences
OS = $(shell uname -s | tr '[:upper:]' '[:lower:]')
BASE64 = base64 -w 0
ifeq ($(OS), darwin)
BASE64 = base64
endif

# go-get-tool will 'go install' any package $1 and install it to LOCAL_BIN.
define go-get-tool
@set -e ;\
Expand Down Expand Up @@ -195,7 +205,7 @@ clean:
-rm build/_output/bin/*
-rm coverage*.out
-rm report*.json
-rm kubeconfig_managed
-rm kubeconfig_managed*
-rm -r vendor/

############################################################
Expand Down Expand Up @@ -235,17 +245,13 @@ kustomize: ## Download kustomize locally if necessary.
GINKGO = $(LOCAL_BIN)/ginkgo

.PHONY: kind-bootstrap-cluster
kind-bootstrap-cluster: kind-create-cluster install-crds kind-deploy-controller install-resources
kind-bootstrap-cluster: kind-bootstrap-cluster-dev kind-deploy-controller

.PHONY: kind-bootstrap-cluster-dev
kind-bootstrap-cluster-dev: kind-create-cluster install-crds install-resources
kind-bootstrap-cluster-dev: kind-create-cluster install-crds kind-controller-kubeconfig

.PHONY: kind-deploy-controller
kind-deploy-controller: generate-operator-yaml
@echo Installing $(IMG)
-kubectl create ns $(KIND_NAMESPACE)
kubectl apply -f deploy/operator.yaml -n $(KIND_NAMESPACE)
kubectl patch deployment $(IMG) -n $(KIND_NAMESPACE) -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$(IMG)\",\"env\":[{\"name\":\"WATCH_NAMESPACE\",\"value\":\"$(WATCH_NAMESPACE)\"}]}]}}}}"
kind-deploy-controller: generate-operator-yaml install-resources deploy

.PHONY: deploy-controller
deploy-controller: kind-deploy-controller
Expand All @@ -262,16 +268,27 @@ kind-deploy-controller-dev: kind-deploy-controller
# Specify KIND_VERSION to indicate the version tag of the KinD image
.PHONY: kind-create-cluster
kind-create-cluster:
@echo "creating cluster"
kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
kind get kubeconfig --name $(KIND_NAME) > $(PWD)/kubeconfig_managed
# ensuring cluster $(KIND_NAME)
-kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
kubectl config use-context $(KIND_CLUSTER_NAME)
kind get kubeconfig --name $(KIND_NAME) > kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e

.PHONY: kind-controller-kubeconfig
kind-controller-kubeconfig: install-resources
kubectl -n $(KIND_NAMESPACE) apply -f test/resources/e2e_controller_secret.yaml
-rm kubeconfig_$(MANAGED_CLUSTER_NAME)
dhaiducek marked this conversation as resolved.
Show resolved Hide resolved
@kubectl config set-cluster $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
--server=$(shell kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}' --kubeconfig=kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e) \
--insecure-skip-tls-verify=true
@kubectl config set-credentials $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
--token=$$(kubectl get secret -n $(KIND_NAMESPACE) config-policy-controller -o jsonpath='{.data.token}' --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME)_e2e | $(BASE64) --decode)
@kubectl config set-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME) \
--user=$(KIND_CLUSTER_NAME) --cluster=$(KIND_CLUSTER_NAME)
@kubectl config use-context $(KIND_CLUSTER_NAME) --kubeconfig=$(PWD)/kubeconfig_$(MANAGED_CLUSTER_NAME)

.PHONY: kind-additional-cluster
kind-additional-cluster:
@echo "creating cluster"
kind create cluster --name $(KIND_NAME)2 $(KIND_ARGS)
kind get kubeconfig --name $(KIND_NAME)2 > $(PWD)/kubeconfig_managed2
kubectl config use-context kind-$(KIND_NAME)
kind-additional-cluster: MANAGED_CLUSTER_SUFFIX = 2
kind-additional-cluster: kind-create-cluster kind-controller-kubeconfig

.PHONY: kind-delete-cluster
kind-delete-cluster:
Expand All @@ -294,8 +311,19 @@ install-crds:

.PHONY: install-resources
install-resources:
@echo creating namespaces
kubectl create ns $(WATCH_NAMESPACE)
# creating namespaces
-kubectl create ns $(WATCH_NAMESPACE)
-kubectl create ns $(KIND_NAMESPACE)
# deploying roles and service account
kubectl apply -k deploy/rbac
kubectl apply -f deploy/manager/service-account.yaml -n $(KIND_NAMESPACE)

.PHONY: kind-ensure-sa
kind-ensure-sa:
@KUBECONFIG_TOKEN="$$(kubectl config view --raw -o jsonpath='{.users[].user.token}')"; \
KUBECONFIG_USER="$$(echo "$${KUBECONFIG_TOKEN}" | jq -rR 'split(".") | .[1] | select(. != null) | @base64d | fromjson | .sub')"; \
echo "Kubeconfig user detected from token: $${KUBECONFIG_USER}"; \
[ "$${KUBECONFIG_USER}" = "system:serviceaccount:$(KIND_NAMESPACE):config-policy-controller" ]

.PHONY: e2e-dependencies
e2e-dependencies:
Expand All @@ -306,7 +334,7 @@ e2e-test: e2e-dependencies
$(GINKGO) -v --fail-fast --slow-spec-threshold=10s $(E2E_TEST_ARGS) test/e2e

.PHONY: e2e-test-coverage
e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter="!hosted-mode" --output-dir=.
e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode' --output-dir=.
e2e-test-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented

.PHONY: e2e-test-hosted-mode-coverage
Expand Down
1 change: 1 addition & 0 deletions deploy/manager/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
resources:
- ./manager.yaml
- ./service-account.yaml
- ../rbac/
18 changes: 0 additions & 18 deletions deploy/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,3 @@ spec:
fieldPath: metadata.name
- name: OPERATOR_NAME
value: "config-policy-controller"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: config-policy-controller
subjects:
- kind: ServiceAccount
name: config-policy-controller
namespace: open-cluster-management-agent-addon
roleRef:
kind: ClusterRole
name: config-policy-controller
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: config-policy-controller
18 changes: 18 additions & 0 deletions deploy/manager/service-account.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: config-policy-controller
subjects:
- kind: ServiceAccount
name: config-policy-controller
namespace: open-cluster-management-agent-addon
roleRef:
kind: ClusterRole
name: config-policy-controller
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: config-policy-controller
4 changes: 2 additions & 2 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ func TestE2e(t *testing.T) {
func init() {
klog.SetOutput(GinkgoWriter)
klog.InitFlags(nil)
flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed",
"Location of the kubeconfig to use; defaults to KUBECONFIG if not set")
flag.StringVar(&kubeconfigManaged, "kubeconfig_managed", "../../kubeconfig_managed_e2e",
"Location of the kubeconfig to use; defaults to current kubeconfig if set to an empty string")
}

var _ = BeforeSuite(func() {
Expand Down
7 changes: 7 additions & 0 deletions test/resources/e2e_controller_secret.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: config-policy-controller
annotations:
kubernetes.io/service-account.name: config-policy-controller