Skip to content

Commit

Permalink
Add integration test for the new cluster labelling logics. (#92)
Browse files Browse the repository at this point in the history
  • Loading branch information
HanFa authored Jun 13, 2022
1 parent b4e6d64 commit 7fc4074
Show file tree
Hide file tree
Showing 14 changed files with 599 additions and 140 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ integration-test: $(GINKGO) $(ETCD)
$(GINKGO) -v controllers/machine -- -enable-integration-tests -enable-unit-tests=false
$(GINKGO) -v controllers/cluster -- -enable-integration-tests -enable-unit-tests=false
$(GINKGO) -v controllers/configmap -- -enable-integration-tests -enable-unit-tests=false
$(GINKGO) -v controllers/tests/cluster_for_akodeploymentconfig/default_adc -- -enable-integration-tests -enable-unit-tests=false -root-dir="../../../.."
$(GINKGO) -v controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors -- -enable-integration-tests -enable-unit-tests=false -root-dir="../../../.."

.PHONY: kind-e2e-test
kind-e2e-test: $(KUSTOMIZE) $(KIND) $(KUBECTL) $(JQ) $(YTT)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,24 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako"
ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
"github.com/vmware/alb-sdk/go/models"
"github.com/vmware/alb-sdk/go/session"
akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/cluster-api/util/conditions"
kcfg "sigs.k8s.io/cluster-api/util/kubeconfig"
ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/util/conditions"
kcfg "sigs.k8s.io/cluster-api/util/kubeconfig"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako"
ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util"
)

func intgTestAkoDeploymentConfigController() {
Expand All @@ -51,56 +52,9 @@ func intgTestAkoDeploymentConfigController() {
userCreateCalled bool
)

staticCluster = &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "integration-test-8ed12g",
Namespace: "integration-test-8ed12g",
},
Spec: clusterv1.ClusterSpec{},
}

defaultAkoDeploymentConfigCommonSpec := akoov1alpha1.AKODeploymentConfigSpec{
DataNetwork: akoov1alpha1.DataNetwork{
Name: "integration-test-8ed12g",
CIDR: "10.0.0.0/24",
IPPools: []akoov1alpha1.IPPool{
{
Start: "10.0.0.1",
End: "10.0.0.10",
Type: "V4",
},
},
},
ControlPlaneNetwork: akoov1alpha1.ControlPlaneNetwork{
Name: "integration-test-8ed12g",
CIDR: "10.1.0.0/24",
},
ServiceEngineGroup: "ha-test",
AdminCredentialRef: &akoov1alpha1.SecretRef{
Name: "controller-credentials",
Namespace: "default",
},
CertificateAuthorityRef: &akoov1alpha1.SecretRef{
Name: "controller-ca",
Namespace: "default",
},
}
akoDeploymentConfigCommonSpec := defaultAkoDeploymentConfigCommonSpec.DeepCopy()
akoDeploymentConfigCommonSpec.ClusterSelector = metav1.LabelSelector{
MatchLabels: map[string]string{
"test": "true",
},
}
staticAkoDeploymentConfig = &akoov1alpha1.AKODeploymentConfig{
ObjectMeta: metav1.ObjectMeta{
Name: "ako-deployment-config",
},
Spec: *akoDeploymentConfigCommonSpec,
}
staticDefaultAkoDeploymentConfig = &akoov1alpha1.AKODeploymentConfig{
ObjectMeta: metav1.ObjectMeta{Name: akoov1alpha1.WorkloadClusterAkoDeploymentConfig},
Spec: defaultAkoDeploymentConfigCommonSpec,
}
staticCluster = &testutil.DefaultCluster
staticAkoDeploymentConfig = testutil.GetCustomizedADC(testutil.CustomizedADCLabels)
staticDefaultAkoDeploymentConfig = testutil.GetDefaultADC()

staticControllerCredentials = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand Down
8 changes: 4 additions & 4 deletions controllers/akodeploymentconfig/phases/phases_unit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
package phases

import (
ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator"

"github.com/go-logr/logr"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"

akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
)

func ReconcilePhaseUnitTest() {
Expand Down
69 changes: 4 additions & 65 deletions controllers/akodeploymentconfig/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,77 +8,16 @@ import (
"testing"

. "github.com/onsi/ginkgo"
akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient"

"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/funcs"
testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util"
akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"

networkv1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"

adccluster "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig/cluster"
cluster "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/cluster"
ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager"
)

// suite is used for unit and integration testing this controller.
var suite = builder.NewTestSuiteForController(
func(mgr ctrlmgr.Manager) error {
rec := &akodeploymentconfig.AKODeploymentConfigReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("AKODeploymentConfig"),
Scheme: mgr.GetScheme(),
}
builder.FakeAvi = aviclient.NewFakeAviClient()
rec.SetAviClient(builder.FakeAvi)

adcClusterReconciler := adccluster.NewReconciler(rec.Client, rec.Log, rec.Scheme)
adcClusterReconciler.GetRemoteClient = adccluster.GetFakeRemoteClient
rec.ClusterReconciler = adcClusterReconciler

if err := rec.SetupWithManager(mgr); err != nil {
return err
}

// involve the cluster controller as well for the resetting skip-default-adc label test
if err := (&cluster.ClusterReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("Cluster"),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
return err
}

return nil
},
func(scheme *runtime.Scheme) (err error) {
err = networkv1alpha1.AddToScheme(scheme)
if err != nil {
return err
}
err = akoov1alpha1.AddToScheme(scheme)
if err != nil {
return err
}
err = akov1alpha1.AddToScheme(scheme)
if err != nil {
return err
}
err = corev1.AddToScheme(scheme)
if err != nil {
return err
}
err = clusterv1.AddToScheme(scheme)
if err != nil {
return err
}
return nil
},
funcs.AddAKODeploymentConfigAndClusterControllerToMgrFunc,
funcs.AddAllToSchemeFunc,
filepath.Join(testutil.FindModuleDir("sigs.k8s.io/cluster-api"), "config", "crd", "bases"),
filepath.Join(testutil.FindModuleDir("github.com/vmware/load-balancer-and-ingress-services-for-kubernetes"), "helm", "ako", "crds"),
)
Expand Down
15 changes: 8 additions & 7 deletions controllers/configmap/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,22 @@ package configmap_test

import (
"context"
"testing"

. "github.com/onsi/ginkgo"
akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/configmap"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"path/filepath"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"
ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager"
"testing"

corev1 "k8s.io/api/core/v1"
akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/configmap"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util"
)

// suite is used for unit and integration testing this controller.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
// Copyright 2022 VMware, Inc.
// SPDX-License-Identifier: Apache-2.0

package default_adc_test

import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"

akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1"
"github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder"
testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util"
)

func intgTestClusterDisableAVIWithoutAnyADC() {
var (
ctx *builder.IntegrationTestContext
staticCluster *clusterv1.Cluster
)

BeforeEach(func() {
ctx = suite.NewIntegrationTestContext()
staticCluster = testutil.GetDefaultCluster()
})

When("there is no ADC and a cluster is created", func() {
BeforeEach(func() {
testutil.CreateObjects(ctx, staticCluster.DeepCopy())
})
AfterEach(func() {
testutil.DeleteObjects(ctx, staticCluster.DeepCopy())
testutil.EnsureRuntimeObjectMatchExpectation(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, &clusterv1.Cluster{}, testutil.NOTFOUND)
})
It("shouldn't have 'networking.tkg.tanzu.vmware.com/avi'", func() {
testutil.EnsureClusterAviLabelExists(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, akoov1alpha1.AviClusterLabel, false)
})
})
}

func intgTestClusterCanBeSelectedByADC() {
var (
ctx *builder.IntegrationTestContext

staticCluster *clusterv1.Cluster
staticAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig
staticDefaultAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig

staticManagementNamespace *v1.Namespace
staticManagementCluster *clusterv1.Cluster
staticManagementAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig
)

BeforeEach(func() {
ctx = suite.NewIntegrationTestContext()
staticCluster = testutil.GetDefaultCluster()
staticAkoDeploymentConfig = testutil.GetCustomizedADC(testutil.CustomizedADCLabels)
staticDefaultAkoDeploymentConfig = testutil.GetDefaultADC()

staticManagementNamespace = &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "tkg-system"},
}
staticManagementCluster = testutil.GetManagementCluster()
staticManagementAkoDeploymentConfig = testutil.GetManagementADC()
})

When("both default and customized ADC exist", func() {

BeforeEach(func() {
testutil.CreateObjects(ctx, staticAkoDeploymentConfig.DeepCopy())
testutil.CreateObjects(ctx, staticDefaultAkoDeploymentConfig.DeepCopy())
testutil.CreateObjects(ctx, staticCluster.DeepCopy())
})

It("labels the cluster dynamically", func() {
By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-all'", func() {
testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, akoov1alpha1.AviClusterLabel, staticDefaultAkoDeploymentConfig.Name)
})

By("add cluster label to use customized adc")
testutil.UpdateObjectLabels(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, testutil.CustomizedADCLabels)

By("labels with 'networking.tkg.tanzu.vmware.com/avi: ako-deployment-config'", func() {
testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, akoov1alpha1.AviClusterLabel, staticAkoDeploymentConfig.Name)
})

By("create another customized ako-deployment-config2")
anotherAkoDeploymentConfig := staticAkoDeploymentConfig.DeepCopy()
anotherAkoDeploymentConfig.Name = "ako-deployment-config-2"
testutil.CreateObjects(ctx, anotherAkoDeploymentConfig.DeepCopy())

By("cluster should keep its label, even through another custom ADC matches the name. a.k.a it won't override", func() {
Consistently(func() bool {
obj := &clusterv1.Cluster{}
err := ctx.Client.Get(ctx.Context, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, obj)
if err != nil {
return false
}
val, ok := obj.Labels[akoov1alpha1.AviClusterLabel]
return ok && val == staticAkoDeploymentConfig.Name
})
})

By("unset cluster label to use default adc")
testutil.UpdateObjectLabels(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, map[string]string{})

By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-all'", func() {
testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{
Name: staticCluster.Name,
Namespace: staticCluster.Namespace,
}, akoov1alpha1.AviClusterLabel, staticDefaultAkoDeploymentConfig.Name)
})
})
})

When("management ADC exists", func() {
BeforeEach(func() {
testutil.CreateObjects(ctx, staticManagementNamespace.DeepCopy())
testutil.CreateObjects(ctx, staticManagementAkoDeploymentConfig.DeepCopy())
testutil.CreateObjects(ctx, staticManagementCluster.DeepCopy())
})

It("labels the management cluster", func() {
By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-management-cluster'", func() {
testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{
Name: staticManagementCluster.Name,
Namespace: staticManagementCluster.Namespace,
}, akoov1alpha1.AviClusterLabel, staticManagementAkoDeploymentConfig.Name)
})
})
})
}
Loading

0 comments on commit 7fc4074

Please sign in to comment.