Skip to content

Commit

Permalink
Add hosted mode tests for OperatorPolicy and ComplianceHistory
Browse files Browse the repository at this point in the history
The "hosting cluster" should be used for the Policy, OperatorPolicy, and compliance events
Ref: https://issues.redhat.com/browse/ACM-11255
Signed-off-by: yiraeChristineKim <yikim@redhat.com>
  • Loading branch information
yiraeChristineKim committed May 8, 2024
1 parent d734b7a commit dfd8690
Show file tree
Hide file tree
Showing 9 changed files with 514 additions and 431 deletions.
131 changes: 86 additions & 45 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -1,48 +1,89 @@
{
"version": "0.2.0",
"configurations": [
// Run `make kind-bootstrap-cluster-dev` before launching this.
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": ["controller", "--leader-elect=false", "--log-level=3", "--v=5", "--enable-operator-policy=true"],
"env": {
"WATCH_NAMESPACE": "managed",
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed",
}
},
// Set FDescribe or FIt on the test to debug. Then set the desired breakpoint.
{
"name": "Launch Test Function (instructions in launch.json)",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/test/e2e/e2e_suite_test.go",
"args": [
"-ginkgo.debug",
"-ginkgo.v",
],
"env": {
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed_e2e",
}
},
// Set the correct path to the governance-policy-framework repo directory in the env section.
{
"name": "Launch Package (Framework E2E) (instructions in launch.json)",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": ["controller", "--leader-elect=false", "--log-level=3", "--v=5", "--enable-operator-policy=true"],
"env": {
"WATCH_NAMESPACE": "managed",
"HUB_CONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_hub",
"MANAGED_CONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_managed",
"KUBECONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_managed",
}
"version": "0.2.0",
"configurations": [
// Run `make kind-bootstrap-cluster-dev` before launching this.
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"controller",
"--leader-elect=false",
"--log-level=3",
"--v=5",
"--enable-operator-policy=true"
],
"env": {
"WATCH_NAMESPACE": "managed",
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed"
}
},
{
"name": "Launch hosted Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"controller",
"--leader-elect=false",
"--log-level=3",
"--v=5",
"--enable-operator-policy=true",
"--target-kubeconfig-path=${workspaceFolder}/kubeconfig_managed2"
],
"env": {
"WATCH_NAMESPACE": "managed",
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed"
}
]
},
// Set FDescribe or FIt on the test to debug. Then set the desired breakpoint.
{
"name": "Launch Test Function (instructions in launch.json)",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/test/e2e/e2e_suite_test.go",
"args": ["-ginkgo.debug", "-ginkgo.v"],
"env": {
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed_e2e"
}
},
// Set FDescribe or FIt on the test to debug. Then set the desired breakpoint.
{
"name": "Launch Hosted Test Function (instructions in launch.json)",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/test/e2e/e2e_suite_test.go",
"args": ["-ginkgo.debug", "-ginkgo.v", "--is_hosted=true"],
"env": {
"KUBECONFIG": "${workspaceFolder}/kubeconfig_managed_e2e",
"TARGET_KUBECONFIG_PATH": "${workspaceFolder}/kubeconfig_managed2_e2e",
}
},
// Set the correct path to the governance-policy-framework repo directory in the env section.
{
"name": "Launch Package (Framework E2E) (instructions in launch.json)",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"controller",
"--leader-elect=false",
"--log-level=3",
"--v=5",
"--enable-operator-policy=true"
],
"env": {
"WATCH_NAMESPACE": "managed",
"HUB_CONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_hub",
"MANAGED_CONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_managed",
"KUBECONFIG": "${userHome}/git/governance-policy-framework/kubeconfig_managed"
}
}
]
}
30 changes: 23 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ create-ns:
# Run against the current locally configured Kubernetes cluster
.PHONY: run
run:
WATCH_NAMESPACE=$(WATCH_NAMESPACE) go run ./main.go controller --leader-elect=false
WATCH_NAMESPACE=$(WATCH_NAMESPACE) go run ./main.go controller --leader-elect=false --enable-operator-policy=true #--target-kubeconfig-path=./kubeconfig_managed2

############################################################
# clean section
Expand Down Expand Up @@ -193,7 +193,7 @@ kind-create-cluster:
.PHONY: kind-additional-cluster
kind-additional-cluster: MANAGED_CLUSTER_SUFFIX = 2
kind-additional-cluster: CLUSTER_NAME = $(MANAGED_CLUSTER_NAME)
kind-additional-cluster: kind-create-cluster kind-controller-kubeconfig
kind-additional-cluster: kind-create-cluster kind-controller-kubeconfig install-crds-hosted

.PHONY: kind-delete-cluster
kind-delete-cluster:
Expand All @@ -204,11 +204,10 @@ kind-delete-cluster:
kind-tests: kind-delete-cluster kind-bootstrap-cluster-dev kind-deploy-controller-dev e2e-test

OLM_VERSION := v0.26.0
OLM_INSTALLER = $(LOCAL_BIN)/install.sh

.PHONY: install-crds
install-crds:
@echo installing olm
curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/$(OLM_VERSION)/install.sh -o $(LOCAL_BIN)/install.sh
install-crds: $(OLM_INSTALLER)
chmod +x $(LOCAL_BIN)/install.sh
$(LOCAL_BIN)/install.sh $(OLM_VERSION)
@echo installing crds
Expand All @@ -221,6 +220,18 @@ install-crds:
kubectl apply -f deploy/crds/policy.open-cluster-management.io_configurationpolicies.yaml
kubectl apply -f deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml



$(OLM_INSTALLER):
@echo installing olm
curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/$(OLM_VERSION)/install.sh -o $(LOCAL_BIN)/install.sh

install-crds-hosted: export KUBECONFIG=./kubeconfig_managed$(MANAGED_CLUSTER_SUFFIX)_e2e
install-crds-hosted: $(OLM_INSTALLER)
chmod +x $(LOCAL_BIN)/install.sh
$(LOCAL_BIN)/install.sh $(OLM_VERSION)


.PHONY: install-resources
install-resources:
# creating namespaces
Expand All @@ -230,17 +241,22 @@ install-resources:
kubectl apply -k deploy/rbac
kubectl apply -f deploy/manager/service-account.yaml -n $(KIND_NAMESPACE)

IS_HOSTED ?= false
E2E_PROCS=20

.PHONY: e2e-test
e2e-test: e2e-dependencies
$(GINKGO) -v --procs=20 $(E2E_TEST_ARGS) test/e2e
$(GINKGO) -v --procs=$(E2E_PROCS) $(E2E_TEST_ARGS) test/e2e -- -is_hosted=$(IS_HOSTED)

.PHONY: e2e-test-coverage
e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode && !running-in-cluster' --output-dir=.
e2e-test-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented

.PHONY: e2e-test-hosted-mode-coverage
e2e-test-hosted-mode-coverage: E2E_TEST_ARGS = --json-report=report_e2e_hosted_mode.json --label-filter="hosted-mode && !running-in-cluster" --output-dir=.
e2e-test-hosted-mode-coverage: E2E_TEST_ARGS = --json-report=report_e2e_hosted_mode.json --label-filter="hosted-mode || hosted-as-well && !running-in-cluster" --output-dir=.
e2e-test-hosted-mode-coverage: COVERAGE_E2E_OUT = coverage_e2e_hosted_mode.out
e2e-test-hosted-mode-coverage: IS_HOSTED=true
e2e-test-hosted-mode-coverage: E2E_PROCS=3
e2e-test-hosted-mode-coverage: export TARGET_KUBECONFIG_PATH = $(PWD)/kubeconfig_managed2
e2e-test-hosted-mode-coverage: e2e-run-instrumented e2e-test e2e-stop-instrumented

Expand Down
23 changes: 12 additions & 11 deletions controllers/operatorpolicy_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type OperatorPolicyReconciler struct {
DynamicWatcher depclient.DynamicWatcher
InstanceName string
DefaultNamespace string
TargetClient client.Client
}

// SetupWithManager sets up the controller with the Manager and will reconcile when the dynamic watcher
Expand Down Expand Up @@ -745,7 +746,7 @@ func (r *OperatorPolicyReconciler) musthaveOpGroup(

desiredOpGroup.ResourceVersion = opGroup.GetResourceVersion()

err = r.Update(ctx, &opGroup)
err = r.TargetClient.Update(ctx, &opGroup)
if err != nil {
return nil, changed, fmt.Errorf("error updating the OperatorGroup: %w", err)
}
Expand All @@ -766,7 +767,7 @@ func (r *OperatorPolicyReconciler) musthaveOpGroup(
func (r *OperatorPolicyReconciler) createWithNamespace(
ctx context.Context, policy *policyv1beta1.OperatorPolicy, object client.Object,
) error {
err := r.Create(ctx, object)
err := r.TargetClient.Create(ctx, object)
if err == nil {
return nil
}
Expand All @@ -784,13 +785,13 @@ func (r *OperatorPolicyReconciler) createWithNamespace(
},
}

err = r.Create(ctx, &ns)
err = r.TargetClient.Create(ctx, &ns)
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}

// Try creating the object again now that the namespace was created.
return r.Create(ctx, object)
return r.TargetClient.Create(ctx, object)
}

// isNamespaceNotFound detects if the input error from r.Create failed due to the specified namespace not existing.
Expand Down Expand Up @@ -899,7 +900,7 @@ func (r *OperatorPolicyReconciler) mustnothaveOpGroup(
earlyConds = append(earlyConds, calculateComplianceCondition(policy))
}

err := r.Delete(ctx, desiredOpGroup)
err := r.TargetClient.Delete(ctx, desiredOpGroup)
if err != nil {
return earlyConds, changed, fmt.Errorf("error deleting the OperatorGroup: %w", err)
}
Expand Down Expand Up @@ -1036,7 +1037,7 @@ func (r *OperatorPolicyReconciler) musthaveSubscription(
earlyConds = append(earlyConds, calculateComplianceCondition(policy))
}

err = r.Update(ctx, foundSub)
err = r.TargetClient.Update(ctx, foundSub)
if err != nil {
return mergedSub, nil, changed, fmt.Errorf("error updating the Subscription: %w", err)
}
Expand Down Expand Up @@ -1090,7 +1091,7 @@ func (r *OperatorPolicyReconciler) mustnothaveSubscription(
earlyConds = append(earlyConds, calculateComplianceCondition(policy))
}

err := r.Delete(ctx, foundUnstructSub)
err := r.TargetClient.Delete(ctx, foundUnstructSub)
if err != nil {
return foundSub, earlyConds, changed, fmt.Errorf("error deleting the Subscription: %w", err)
}
Expand Down Expand Up @@ -1354,7 +1355,7 @@ func (r *OperatorPolicyReconciler) musthaveInstallPlan(
return false, fmt.Errorf("error approving InstallPlan: %w", err)
}

if err := r.Update(ctx, &approvableInstallPlans[0]); err != nil {
if err := r.TargetClient.Update(ctx, &approvableInstallPlans[0]); err != nil {
return false, fmt.Errorf("error updating approved InstallPlan: %w", err)
}

Expand Down Expand Up @@ -1492,7 +1493,7 @@ func (r *OperatorPolicyReconciler) mustnothaveCSV(
continue
}

err := r.Delete(ctx, &csvList[i])
err := r.TargetClient.Delete(ctx, &csvList[i])
if err != nil {
changed := updateStatus(policy, foundNotWantedCond("ClusterServiceVersion", csvNames...), relatedCSVs...)

Expand Down Expand Up @@ -1662,7 +1663,7 @@ func (r *OperatorPolicyReconciler) handleCRDs(
continue
}

err := r.Delete(ctx, &crdList[i])
err := r.TargetClient.Delete(ctx, &crdList[i])
if err != nil {
changed := updateStatus(policy, foundNotWantedCond("CustomResourceDefinition"), relatedCRDs...)

Expand Down Expand Up @@ -1825,7 +1826,7 @@ func (r *OperatorPolicyReconciler) mergeObjects(
}

if updateNeeded {
err := r.Update(ctx, existing, client.DryRunAll)
err := r.TargetClient.Update(ctx, existing, client.DryRunAll)
if err != nil {
if k8serrors.IsForbidden(err) {
// This indicates the update would make a change, but the change is not allowed,
Expand Down
10 changes: 9 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,13 +322,15 @@ func main() {
var targetK8sClient kubernetes.Interface
var targetK8sDynamicClient dynamic.Interface
var targetK8sConfig *rest.Config
var targetClient client.Client
var nsSelMgr manager.Manager // A separate controller-manager is needed in hosted mode

if opts.targetKubeConfig == "" {
targetK8sConfig = cfg
targetK8sClient = kubernetes.NewForConfigOrDie(targetK8sConfig)
targetK8sDynamicClient = dynamic.NewForConfigOrDie(targetK8sConfig)
nsSelMgr = mgr
targetClient = mgr.GetClient()
} else { // "Hosted mode"
var err error

Expand All @@ -343,6 +345,11 @@ func main() {

targetK8sClient = kubernetes.NewForConfigOrDie(targetK8sConfig)
targetK8sDynamicClient = dynamic.NewForConfigOrDie(targetK8sConfig)
targetClient, err = client.New(targetK8sConfig, client.Options{Scheme: scheme})
if err != nil {
log.Error(err, "Failed to load the target kubeconfig", "path", opts.targetKubeConfig)
os.Exit(1)
}

// The managed cluster's API server is potentially not the same as the hosting cluster and it could be
// offline already as part of the uninstall process. In this case, the manager's instantiation will fail.
Expand Down Expand Up @@ -428,7 +435,7 @@ func main() {
if opts.enableOperatorPolicy {
depReconciler, depEvents := depclient.NewControllerRuntimeSource()

watcher, err := depclient.New(cfg, depReconciler,
watcher, err := depclient.New(targetK8sConfig, depReconciler,
&depclient.Options{
DisableInitialReconcile: true,
EnableCache: true,
Expand All @@ -455,6 +462,7 @@ func main() {
DynamicWatcher: watcher,
InstanceName: instanceName,
DefaultNamespace: opts.operatorPolDefaultNS,
TargetClient: targetClient,
}

if err = OpReconciler.SetupWithManager(mgr, depEvents); err != nil {
Expand Down
Loading

0 comments on commit dfd8690

Please sign in to comment.