add functional test for instascale machineset test case in accordance…

… with using envtest control plane or using existing-cluster

update

update

.github/workflows/functional-test.yml

@@ -0,0 +1,60 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Functional-tests
+
+on:
+ push:
+ branches: [ "main" ]
+ pull_request:
+ branches: [ "main" ]
+
+jobs:
+
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set Go
+ uses: actions/setup-go@v3
+ with:
+ go-version: v1.19
+
+ - name: init directories
+ run: mkdir -p "$(pwd)/bin"
+
+ - name: Download kubebuilder 
+ run: |
+ version=1.0.8
+ arch=amd64
+ curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${version}/kubebuilder_${version}_linux_${arch}.tar.gz"
+ tar -zxvf kubebuilder_${version}_linux_${arch}.tar.gz
+ mv kubebuilder_${version}_linux_${arch} kubebuilder
+ echo "kubebuilder_${version}_linux_${arch} renamed to kubebuilder"
+ sudo mv kubebuilder /usr/local/
+ echo "kubebuilder moved to /usr/local/ path"
+ DIRECTORY=/usr/local/kubebuilder
+ if [ -d "$DIRECTORY" ]; then
+ echo "$DIRECTORY does exist."
+ else
+ echo "$DIRECTORY does not exist."
+ fi
+ echo "PATH=$PATH:$DIRECTORY/bin" >> $GITHUB_ENV
+
+ - name: Set up Kubebuilder and setup-envtest
+ run: |
+ make envtest
+ echo "KUBEBUILDER_ASSETS=$(./bin/setup-envtest use 1.23 -p path)" >> $GITHUB_ENV
+ echo "USE_EXISTING_CLUSTER=false" >> $GITHUB_ENV
+ echo "KUBEBUILDER_ASSETS : $(./bin/setup-envtest use 1.23 -p path)"
+ echo "USE_EXISTING_CLUSTER : false"
+ 
+ - name: Test
+ run: go test -v ./functional-tests
+
+ - name: Upload test log-file as an artifact
+ uses: actions/upload-artifact@v2
+ with: 
+ name: functional-test-logs
+ path: ./functional-tests/functional-test-logfile.log

.github/workflows/go.yml

@@ -25,4 +25,4 @@ jobs:
  run: go build -o bin/manager main.go
 
  - name: Test
- run: go test -v ./...
+ run: go test -v ./controllers

.gitignore

@@ -25,4 +25,7 @@ testbin/*
 *~
 
 # VS Code files
-.vscode
+.vscode
+
+# Functional-test generated log-file
+*.log

functional-tests/appwrapper_reconciler_test.go

@@ -0,0 +1,228 @@
+package functional_tests
+
+import (
+ "context"
+ "go/build"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ gstruct "github.com/onsi/gomega/gstruct"
+ machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+ . "github.com/project-codeflare/codeflare-common/support"
+ "github.com/project-codeflare/instascale/controllers"
+ "github.com/project-codeflare/instascale/pkg/config"
+ mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+ apiv1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/scheme"
+ "k8s.io/client-go/rest"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/envtest"
+ log "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+var (
+ ctx context.Context
+ cancel context.CancelFunc
+)
+
+func logger() {
+ // Get log file path from environment variable or use a default path
+ logFilePath := os.Getenv("LOG_FILE_PATH")
+ if logFilePath == "" {
+ logFilePath = "./functional-test-logfile.log"
+ }
+ // Create a log file
+ logFile, err := os.Create(logFilePath)
+ if err != nil {
+ log.Log.Error(err, "Error creating log file: %v", err)
+ }
+ // Configure zap logger to write to the log file
+ logger := zap.New(zap.WriteTo(logFile), zap.UseDevMode(true))
+
+ // Set the logger for controller-runtime
+ ctrl.SetLogger(logger)
+
+ // This line prevents controller-runtime from complaining about log.SetLogger never being called
+ log.SetLogger(logger)
+}
+
+func startEnvTest(t *testing.T) *rest.Config {
+ // to redirect all functional test related logs to separate logfile ~ default (in functional-tests directory), can be changed using environment variable LOG_FILE_PATH=/path_to_logfile
+ logger()
+
+ //specify testEnv configuration
+ testEnv := &envtest.Environment{
+ CRDDirectoryPaths: []string{
+ filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "project-codeflare", "multi-cluster-app-dispatcher@v1.39.0", "config", "crd", "bases"),
+ filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "openshift", "api@v0.0.0-20230213134911-7ba313770556", "machine", "v1beta1"),
+ },
+ ErrorIfCRDPathMissing: true,
+ }
+ test := WithConfig(t, testEnv.Config)
+ cfg, err := testEnv.Start()
+ test.Expect(err).NotTo(HaveOccurred())
+ test.Expect(cfg).NotTo(BeNil())
+
+ t.Cleanup(func() {
+ teardownTestEnv(test, testEnv)
+ })
+
+ return cfg
+}
+
+func startInstascaleController(test Test, cfg *rest.Config) {
+ ctx, cancel = context.WithCancel(context.TODO())
+
+ // Add custom resource schemes to the global scheme.
+ err := mcadv1beta1.AddToScheme(scheme.Scheme)
+ test.Expect(err).NotTo(HaveOccurred())
+
+ err = machinev1beta1.AddToScheme(scheme.Scheme)
+ test.Expect(err).NotTo(HaveOccurred())
+
+ // Create a controller manager for managing controllers.
+ k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
+ Scheme: scheme.Scheme,
+ })
+ test.Expect(err).ToNot(HaveOccurred())
+
+ // Create an instance of the AppWrapperReconciler with configuration
+ instaScaleController := &controllers.AppWrapperReconciler{
+ Client: k8sManager.GetClient(),
+ Scheme: k8sManager.GetScheme(),
+ Config: config.InstaScaleConfiguration{
+ MachineSetsStrategy: "reuse",
+ MaxScaleoutAllowed: 5,
+ },
+ }
+ // Set up the AppWrapperReconciler with the manager.
+ err = instaScaleController.SetupWithManager(context.Background(), k8sManager)
+ test.Expect(err).ToNot(HaveOccurred())
+
+ // Start the controller manager in a goroutine.
+ go func() {
+ defer GinkgoRecover()
+ err = k8sManager.Start(ctx)
+ test.Expect(err).ToNot(HaveOccurred())
+ }()
+
+}
+
+func teardownTestEnv(test Test, testEnv *envtest.Environment) {
+ cancel()
+ By("tearing down the test environment")
+ err := testEnv.Stop()
+ test.Expect(err).NotTo(HaveOccurred())
+}
+
+func instascaleAppwrapper(namespace string) *mcadv1beta1.AppWrapper {
+ aw := &mcadv1beta1.AppWrapper{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-instascale",
+ Namespace: namespace,
+ Labels: map[string]string{
+ "orderedinstance": "test.instance1",
+ },
+ Finalizers: []string{"instascale.codeflare.dev/finalizer"},
+ },
+ Spec: mcadv1beta1.AppWrapperSpec{
+ AggrResources: mcadv1beta1.AppWrapperResourceList{
+ GenericItems: []mcadv1beta1.AppWrapperGenericResource{
+ {
+ DesiredAvailable: 1,
+ CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{
+ {
+ Replicas: 1,
+ Requests: apiv1.ResourceList{
+ apiv1.ResourceCPU: resource.MustParse("250m"),
+ apiv1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ Limits: apiv1.ResourceList{
+ apiv1.ResourceCPU: resource.MustParse("1"),
+ apiv1.ResourceMemory: resource.MustParse("1G"),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+ return aw
+}
+
+func TestReconciler(t *testing.T) {
+ // initiate setting up the EnvTest environment
+ cfg := startEnvTest(t)
+
+ test := WithConfig(t, cfg)
+ startInstascaleController(test, cfg) // setup client required for test to interact with kubernetes machine API
+
+ // initialize a variable with test client
+ client := test.Client()
+
+ //read machineset yaml from file `test_instascale_machineset.yml`
+ b, err := os.ReadFile("test_instascale_machineset.yml")
+ test.Expect(err).ToNot(HaveOccurred())
+
+ //deserialize kubernetes object
+ decode := scheme.Codecs.UniversalDeserializer().Decode
+ ms, _, err := decode(b, nil, nil) //decode machineset content of YAML file into kubernetes object
+ test.Expect(err).ToNot(HaveOccurred())
+ msa := ms.(*machinev1beta1.MachineSet) //asserts that decoded object is of type `*machinev1beta1.MachineSet`
+
+ //create machineset in default namespace
+ ms, err = client.Machine().MachineV1beta1().MachineSets("default").Create(test.Ctx(), msa, metav1.CreateOptions{})
+ test.Expect(err).ToNot(HaveOccurred())
+
+ //assert that the replicas in Machineset specification is 0, before appwrapper creation
+ test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(0)))))
+
+ //create new test namespace
+ namespace := test.NewTestNamespace()
+
+ // initializes an appwrapper in the created namespace
+ aw := instascaleAppwrapper(namespace.Name)
+
+ // create appwrapper resource using mcadClient
+ aw, err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
+ test.Expect(err).ToNot(HaveOccurred())
+
+ //update appwrapper status to avoid appwrapper dispatch in case of Insufficient resources
+ aw.Status = mcadv1beta1.AppWrapperStatus{
+ Pending: 1,
+ State: mcadv1beta1.AppWrapperStateEnqueued,
+ Conditions: []mcadv1beta1.AppWrapperCondition{
+ {
+ Type: mcadv1beta1.AppWrapperCondBackoff,
+ Status: apiv1.ConditionTrue,
+ Reason: "AppWrapperNotRunnable",
+ Message: "Insufficient resources to dispatch AppWrapper.",
+ },
+ },
+ }
+ _, err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).UpdateStatus(test.Ctx(), aw, metav1.UpdateOptions{})
+ test.Expect(err).ToNot(HaveOccurred())
+
+ // assert for machine replicas belonging to the machine set after appwrapper creation- there should be 1
+ test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(1)))))
+
+ // delete appwrapper
+ err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{})
+ test.Expect(err).ToNot(HaveOccurred())
+
+ // assert for machine belonging to the machine set after appwrapper deletion- there should be none
+ test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(0)))))
+
+ // delete machineset
+ err = client.Machine().MachineV1beta1().MachineSets("default").Delete(test.Ctx(), "test-instascale", metav1.DeleteOptions{})
+ test.Expect(err).ToNot(HaveOccurred())
+
+}

functional-tests/test_instascale_machineset.yml

@@ -0,0 +1,63 @@
+apiVersion: machine.openshift.io/v1beta1
+kind: MachineSet
+metadata:
+ name: test-instascale
+ labels:
+ machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+spec:
+ replicas: 0
+ selector:
+ matchLabels:
+ machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+ machine.openshift.io/cluster-api-machineset: test-instascale
+ template:
+ metadata:
+ labels:
+ machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+ machine.openshift.io/cluster-api-machine-role: worker
+ machine.openshift.io/cluster-api-machine-type: worker
+ machine.openshift.io/cluster-api-machineset: test-instascale
+ spec:
+ lifecycleHooks: {}
+ metadata:
+ labels:
+ node-role.kubernetes.io/<role>: ''
+ providerSpec:
+ value:
+ userDataSecret:
+ name: worker-user-data
+ placement:
+ availabilityZone: us-east-1a
+ region: us-east-1
+ credentialsSecret:
+ name: aws-cloud-credentials
+ instanceType: test.instance1
+ metadata:
+ creationTimestamp: null
+ blockDevices:
+ - ebs:
+ iops: 0
+ kmsKey: {}
+ volumeSize: 120
+ volumeType: gp2
+ securityGroups:
+ - filters:
+ - name: 'tag:Name'
+ values:
+ - fionawaters-v2pm4-worker-sg
+ kind: AWSMachineProviderConfig
+ metadataServiceOptions: {}
+ tags:
+ - name: kubernetes.io/cluster/fionawaters-v2pm4
+ value: owned
+ deviceIndex: 0
+ ami:
+ id: ami-0624891c612b5eaa0
+ subnet:
+ filters:
+ - name: 'tag:Name'
+ values:
+ - fionawaters-v2pm4-private-us-east-1a
+ apiVersion: awsproviderconfig.openshift.io/v1beta1
+ iamInstanceProfile:
+ id: fionawaters-v2pm4-worker-profile