Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add functional tests #193

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions .github/workflows/functional-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# This workflow will build a golang project
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go

name: Functional-tests

on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]

jobs:

build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: Set Go
uses: actions/setup-go@v3
with:
go-version: v1.19

abhijeet-dhumal marked this conversation as resolved.
Show resolved Hide resolved
- name: init directories
run: mkdir -p "$(pwd)/bin"

- name: Download kubebuilder
run: |
version=1.0.8
arch=amd64
curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${version}/kubebuilder_${version}_linux_${arch}.tar.gz"
tar -zxvf kubebuilder_${version}_linux_${arch}.tar.gz
mv kubebuilder_${version}_linux_${arch} kubebuilder
echo "kubebuilder_${version}_linux_${arch} renamed to kubebuilder"
sudo mv kubebuilder /usr/local/
echo "kubebuilder moved to /usr/local/ path"
DIRECTORY=/usr/local/kubebuilder
if [ -d "$DIRECTORY" ]; then
echo "$DIRECTORY does exist."
else
echo "$DIRECTORY does not exist."
fi
echo "PATH=$PATH:$DIRECTORY/bin" >> $GITHUB_ENV

- name: Set up Kubebuilder and setup-envtest
run: |
make envtest
echo "KUBEBUILDER_ASSETS=$(./bin/setup-envtest use 1.23 -p path)" >> $GITHUB_ENV
echo "USE_EXISTING_CLUSTER=false" >> $GITHUB_ENV
echo "KUBEBUILDER_ASSETS : $(./bin/setup-envtest use 1.23 -p path)"
echo "USE_EXISTING_CLUSTER : false"

- name: Test
run: go test -v ./functional-tests

- name: Upload test log-file as an artifact
uses: actions/upload-artifact@v2
with:
name: functional-test-logs
path: ./functional-tests/functional-test-logfile.log
2 changes: 1 addition & 1 deletion .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ jobs:
run: go build -o bin/manager main.go

- name: Test
run: go test -v ./...
run: go test -v ./controllers
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,7 @@ testbin/*
*~

# VS Code files
.vscode
.vscode

# Functional-test generated log-file
*.log
225 changes: 225 additions & 0 deletions functional-tests/appwrapper_reconciler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,225 @@
package functional_tests

import (
"context"
"go/build"
"os"
"path/filepath"
"testing"
"time"

. "github.com/onsi/gomega"
gstruct "github.com/onsi/gomega/gstruct"
machinev1beta1 "github.com/openshift/api/machine/v1beta1"
. "github.com/project-codeflare/codeflare-common/support"
"github.com/project-codeflare/instascale/controllers"
"github.com/project-codeflare/instascale/pkg/config"
mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/envtest"
log "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

var (
ctx context.Context
cancel context.CancelFunc
)

func logger() {
// Get log file path from environment variable or use a default path
logFilePath := os.Getenv("LOG_FILE_PATH")
if logFilePath == "" {
logFilePath = "./functional-test-logfile.log"
sutaakar marked this conversation as resolved.
Show resolved Hide resolved
}
// Create a log file
logFile, err := os.Create(logFilePath)
if err != nil {
log.Log.Error(err, "Error creating log file: %v", err)
}
// Configure zap logger to write to the log file
logger := zap.New(zap.WriteTo(logFile), zap.UseDevMode(true))

// Set the logger for controller-runtime
ctrl.SetLogger(logger)

// This line prevents controller-runtime from complaining about log.SetLogger never being called
log.SetLogger(logger)
}

func startEnvTest(t *testing.T) *rest.Config {
// to redirect all functional test related logs to separate logfile ~ default (in functional-tests directory), can be changed using environment variable LOG_FILE_PATH=/path_to_logfile
logger()

//specify testEnv configuration
testEnv := &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "project-codeflare", "multi-cluster-app-dispatcher@v1.39.0", "config", "crd", "bases"),
filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "openshift", "api@v0.0.0-20230213134911-7ba313770556", "machine", "v1beta1"),
},
abhijeet-dhumal marked this conversation as resolved.
Show resolved Hide resolved
ErrorIfCRDPathMissing: true,
}
test := WithConfig(t, testEnv.Config)
cfg, err := testEnv.Start()
test.Expect(err).NotTo(HaveOccurred())
test.Expect(cfg).NotTo(BeNil())

t.Cleanup(func() {
teardownTestEnv(test, testEnv)
})

return cfg
}

func startInstascaleController(test Test, cfg *rest.Config) {
ctx, cancel = context.WithCancel(context.TODO())

// Add custom resource schemes to the global scheme.
err := mcadv1beta1.AddToScheme(scheme.Scheme)
test.Expect(err).NotTo(HaveOccurred())

err = machinev1beta1.AddToScheme(scheme.Scheme)
test.Expect(err).NotTo(HaveOccurred())

// Create a controller manager for managing controllers.
k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
Scheme: scheme.Scheme,
})
test.Expect(err).ToNot(HaveOccurred())

// Create an instance of the AppWrapperReconciler with configuration
instaScaleController := &controllers.AppWrapperReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
Config: config.InstaScaleConfiguration{
MachineSetsStrategy: "reuse",
MaxScaleoutAllowed: 5,
},
}
// Set up the AppWrapperReconciler with the manager.
err = instaScaleController.SetupWithManager(context.Background(), k8sManager)
test.Expect(err).ToNot(HaveOccurred())

// Start the controller manager in a goroutine.
go func() {
err = k8sManager.Start(ctx)
test.Expect(err).ToNot(HaveOccurred())
}()

}

func teardownTestEnv(test Test, testEnv *envtest.Environment) {
cancel()
err := testEnv.Stop()
test.Expect(err).NotTo(HaveOccurred())
}

func instascaleAppwrapper(namespace string) *mcadv1beta1.AppWrapper {
aw := &mcadv1beta1.AppWrapper{
ObjectMeta: metav1.ObjectMeta{
Name: "test-instascale",
Namespace: namespace,
Labels: map[string]string{
"orderedinstance": "test.instance1",
},
Finalizers: []string{"instascale.codeflare.dev/finalizer"},
},
Spec: mcadv1beta1.AppWrapperSpec{
AggrResources: mcadv1beta1.AppWrapperResourceList{
GenericItems: []mcadv1beta1.AppWrapperGenericResource{
{
DesiredAvailable: 1,
CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{
{
Replicas: 1,
Requests: apiv1.ResourceList{
apiv1.ResourceCPU: resource.MustParse("250m"),
apiv1.ResourceMemory: resource.MustParse("512Mi"),
},
Limits: apiv1.ResourceList{
apiv1.ResourceCPU: resource.MustParse("1"),
apiv1.ResourceMemory: resource.MustParse("1G"),
},
},
},
},
},
},
},
}
return aw
}

func TestReconciler(t *testing.T) {
// initiate setting up the EnvTest environment
cfg := startEnvTest(t)

test := WithConfig(t, cfg)
startInstascaleController(test, cfg) // setup client required for test to interact with kubernetes machine API

// initialize a variable with test client
client := test.Client()

//read machineset yaml from file `test_instascale_machineset.yml`
b, err := os.ReadFile("test_instascale_machineset.yml")
test.Expect(err).ToNot(HaveOccurred())

//deserialize kubernetes object
decode := scheme.Codecs.UniversalDeserializer().Decode
ms, _, err := decode(b, nil, nil) //decode machineset content of YAML file into kubernetes object
test.Expect(err).ToNot(HaveOccurred())
msa := ms.(*machinev1beta1.MachineSet) //asserts that decoded object is of type `*machinev1beta1.MachineSet`

//create machineset in default namespace
ms, err = client.Machine().MachineV1beta1().MachineSets("default").Create(test.Ctx(), msa, metav1.CreateOptions{})
test.Expect(err).ToNot(HaveOccurred())

//assert that the replicas in Machineset specification is 0, before appwrapper creation
test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(0)))))

//create new test namespace
namespace := test.NewTestNamespace()

// initializes an appwrapper in the created namespace
aw := instascaleAppwrapper(namespace.Name)

// create appwrapper resource using mcadClient
aw, err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
test.Expect(err).ToNot(HaveOccurred())

//update appwrapper status to avoid appwrapper dispatch in case of Insufficient resources
aw.Status = mcadv1beta1.AppWrapperStatus{
Pending: 1,
State: mcadv1beta1.AppWrapperStateEnqueued,
Conditions: []mcadv1beta1.AppWrapperCondition{
{
Type: mcadv1beta1.AppWrapperCondBackoff,
Status: apiv1.ConditionTrue,
Reason: "AppWrapperNotRunnable",
Message: "Insufficient resources to dispatch AppWrapper.",
},
},
}
_, err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).UpdateStatus(test.Ctx(), aw, metav1.UpdateOptions{})
test.Expect(err).ToNot(HaveOccurred())

// assert for machine replicas belonging to the machine set after appwrapper creation- there should be 1
test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(1)))))

// delete appwrapper
err = client.MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{})
test.Expect(err).ToNot(HaveOccurred())

abhijeet-dhumal marked this conversation as resolved.
Show resolved Hide resolved
// assert for machine belonging to the machine set after appwrapper deletion- there should be none
test.Eventually(MachineSet(test, "default", "test-instascale"), 10*time.Second).Should(WithTransform(MachineSetReplicas, gstruct.PointTo(Equal(int32(0)))))

// delete machineset
err = client.Machine().MachineV1beta1().MachineSets("default").Delete(test.Ctx(), "test-instascale", metav1.DeleteOptions{})
test.Expect(err).ToNot(HaveOccurred())

}
63 changes: 63 additions & 0 deletions functional-tests/test_instascale_machineset.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
apiVersion: machine.openshift.io/v1beta1
kind: MachineSet
metadata:
name: test-instascale
labels:
machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
spec:
replicas: 0
selector:
matchLabels:
machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
machine.openshift.io/cluster-api-machineset: test-instascale
template:
metadata:
labels:
machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
machine.openshift.io/cluster-api-machine-role: worker
machine.openshift.io/cluster-api-machine-type: worker
machine.openshift.io/cluster-api-machineset: test-instascale
spec:
lifecycleHooks: {}
metadata:
labels:
node-role.kubernetes.io/<role>: ''
providerSpec:
value:
userDataSecret:
name: worker-user-data
placement:
availabilityZone: us-east-1a
region: us-east-1
credentialsSecret:
name: aws-cloud-credentials
instanceType: test.instance1
metadata:
creationTimestamp: null
blockDevices:
- ebs:
iops: 0
kmsKey: {}
volumeSize: 120
volumeType: gp2
securityGroups:
- filters:
- name: 'tag:Name'
values:
- fionawaters-v2pm4-worker-sg
kind: AWSMachineProviderConfig
metadataServiceOptions: {}
tags:
- name: kubernetes.io/cluster/fionawaters-v2pm4
value: owned
deviceIndex: 0
ami:
id: ami-0624891c612b5eaa0
subnet:
filters:
- name: 'tag:Name'
values:
- fionawaters-v2pm4-private-us-east-1a
apiVersion: awsproviderconfig.openshift.io/v1beta1
iamInstanceProfile:
id: fionawaters-v2pm4-worker-profile
Loading
Loading