diff --git a/src/cloud-api-adaptor/Makefile b/src/cloud-api-adaptor/Makefile index 5ea540f0e..e5eb976ce 100644 --- a/src/cloud-api-adaptor/Makefile +++ b/src/cloud-api-adaptor/Makefile @@ -132,12 +132,22 @@ else endif ifeq ($(RESOURCE_CTRL),true) $(MAKE) -C ../peerpod-ctrl deploy + @if kubectl get svc -n cert-manager cert-manager-webhook > /dev/null 2>&1; then \ + echo "Installing webhook" ;\ + $(MAKE) -C ../webhook deploy ;\ + else \ + echo "Installing cert-manager" ;\ + $(MAKE) -C ../webhook deploy-cert-manager ;\ + echo "Installing webhook" ;\ + $(MAKE) -C ../webhook deploy ;\ + fi endif .PHONY: delete delete: ## Delete cloud-api-adaptor using the operator, according to install/overlays/$(CLOUD_PROVIDER)/kustomization.yaml file. ifeq ($(RESOURCE_CTRL),true) $(MAKE) -C ../peerpod-ctrl undeploy + $(MAKE) -C ../webhook undeploy endif ifneq ($(CLOUD_PROVIDER),) kubectl delete -k install/overlays/$(CLOUD_PROVIDER) diff --git a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go index 84334f88b..ab3b0ed20 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go @@ -15,7 +15,9 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "sigs.k8s.io/e2e-framework/klient" @@ -332,6 +334,26 @@ func IsBufferEmpty(buffer bytes.Buffer) bool { } } +func AssessPodRequestAndLimit(ctx context.Context, client klient.Client, pod *v1.Pod) error { + // Check if the pod has the "kata.peerpods.io/vm request and limit with value "1" + + podVmExtResource := "kata.peerpods.io/vm" + + request := pod.Spec.Containers[0].Resources.Requests[corev1.ResourceName(podVmExtResource)] + limit := pod.Spec.Containers[0].Resources.Limits[corev1.ResourceName(podVmExtResource)] + + // Check if the request and limit are set to "1" + if request.Cmp(resource.MustParse("1")) != 0 { + return fmt.Errorf("request for podvm extended resource is not set to 1") + } + if limit.Cmp(resource.MustParse("1")) != 0 { + return fmt.Errorf("limit for podvm extended resource is not set to 1") + } + + return nil + +} + func AssessPodTestCommands(ctx context.Context, client klient.Client, pod *v1.Pod, testCommands []TestCommand) (string, error) { var podlist v1.PodList if err := client.Resources(pod.Namespace).List(ctx, &podlist); err != nil { diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index a1d28322b..52c6d527b 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -439,6 +439,12 @@ func (tc *TestCase) Run() { } } + err := AssessPodRequestAndLimit(ctx, client, tc.pod) + if err != nil { + t.Logf("request and limit for podvm extended resource are not set to 1") + t.Fatal(err) + } + tc.assert.HasPodVM(t, tc.pod.Name) } diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index d9556937a..c963f36f2 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -152,6 +152,28 @@ func WithCommand(command []string) PodOption { } } +func WithCpuMemRequestAndLimit(cpuRequest, memRequest, cpuLimit, memLimit string) PodOption { + // If any of the parameters is empty, don't set it + return func(p *corev1.Pod) { + p.Spec.Containers[0].Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, + Limits: corev1.ResourceList{}, + } + if cpuRequest != "" { + p.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse(cpuRequest) + } + if memRequest != "" { + p.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse(memRequest) + } + if cpuLimit != "" { + p.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = resource.MustParse(cpuLimit) + } + if memLimit != "" { + p.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = resource.MustParse(memLimit) + } + } +} + type JobOption func(*batchv1.Job) func WithJobCommand(command []string) JobOption { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index c575c8103..035e8ae47 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -751,3 +751,18 @@ func DoTestPodWithSpecificCommands(t *testing.T, e env.Environment, assert Cloud NewTestCase(t, e, "PodWithSpecificCommands", assert, "Pod with specific commands").WithPod(pod).WithTestCommands(testCommands).Run() } + +// Test to run a pod with cpu and memory limits and requests +func DoTestPodWithCpuMemLimitsAndRequests(t *testing.T, e env.Environment, assert CloudAssert, cpuRequest, memRequest, cpuLimit, memLimit string) { + + pod := NewPod(E2eNamespace, "pod-with-cpu-mem-limits-requests", "busybox", BUSYBOX_IMAGE, + WithCpuMemRequestAndLimit(cpuRequest, memRequest, cpuLimit, memLimit)) + + // Add testCommands to check that request/limit are removed from the spec and following annotations + // to pod spec is added + // io.katacontainers.config.hypervisor.default_cpus + // io.katacontainers.config.hypervisor.default_memory + // Custom resource added as req/limit - "kata.peerpods.io/vm + + NewTestCase(t, e, "PodWithCpuMemLimitsAndRequests", assert, "Pod with cpu and memory limits and requests").WithPod(pod).Run() +} diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go index e777b805f..4c772267d 100644 --- a/src/cloud-api-adaptor/test/e2e/docker_test.go +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -129,3 +129,29 @@ func TestDockerCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testin t.Skip("Registry Credentials, or authenticated image name not exported") } } + +func TestDockerCreateWithCpuLimit(t *testing.T) { + // This test is covered as part of unit test and hence skipping to optimise CI time + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestPodWithCpuMemLimitsAndRequests(t, testEnv, assert, "", "", "200m", "") +} + +func TestDockerCreateWithMemLimit(t *testing.T) { + // This test is covered as part of unit test and hence skipping to optimise CI time + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestPodWithCpuMemLimitsAndRequests(t, testEnv, assert, "", "", "", "200Mi") +} + +func TestDockerCreateWithCpuAndMemLimit(t *testing.T) { + // This test is covered as part of unit test and hence skipping to optimise CI time + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestPodWithCpuMemLimitsAndRequests(t, testEnv, assert, "", "", "200m", "200Mi") +} + +func TestDockerCreateWithCpuAndMemRequestLimit(t *testing.T) { + assert := DockerAssert{} + DoTestPodWithCpuMemLimitsAndRequests(t, testEnv, assert, "100m", "100Mi", "200m", "200Mi") +} diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 0c6e17f31..5f972b010 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -165,3 +165,8 @@ func TestLibvirtCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testi t.Skip("Registry Credentials, or authenticated image name not exported") } } + +func TestLibvirtCreateWithCpuAndMemRequestLimit(t *testing.T) { + assert := LibvirtAssert{} + DoTestPodWithCpuMemLimitsAndRequests(t, testEnv, assert, "100m", "100Mi", "200m", "200Mi") +} diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 7001c2b4b..3c69b8080 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -322,6 +322,40 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props return err } + log.Info("Installing cert-manager") + cmd = exec.Command("make", "-C", "../webhook", "deploy-cert-manager") + // Run the deployment from the root src dir + cmd.Dir = p.rootSrcDir + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err = cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + log.Infof("Error in install cert-manager: %s: %s", err, stdoutStderr) + + return err + } + + log.Info("Installing webhook") + cmd = exec.Command("make", "-C", "../webhook", "deploy") + // Run the deployment from the root src dir + cmd.Dir = p.rootSrcDir + // Set the KUBECONFIG env var + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err = cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + + // Wait for the webhook deployment to be ready + log.Info("Wait for the webhook deployment to be available") + if err = wait.For(conditions.New(resources).DeploymentConditionMatch( + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "peer-pods-webhook-controller-manager", Namespace: "peer-pods-webhook-system"}}, + appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*5)); err != nil { + return err + } + return nil } diff --git a/src/webhook/Makefile b/src/webhook/Makefile index fadd1d7b0..207a0db91 100644 --- a/src/webhook/Makefile +++ b/src/webhook/Makefile @@ -129,8 +129,8 @@ deploy-cert-manager: ## Deploy cert-manager for webhook. # Deploy cert-manager kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.15.3/cert-manager.yaml # Wait for service to be up - kubectl wait --timeout=30s -n cert-manager endpoints/cert-manager --for=jsonpath='{.subsets[0].addresses[0].ip}' - kubectl wait --timeout=30s -n cert-manager endpoints/cert-manager-webhook --for=jsonpath='{.subsets[0].addresses[0].ip}' + kubectl wait --timeout=90s -n cert-manager endpoints/cert-manager --for=jsonpath='{.subsets[0].addresses[0].ip}' + kubectl wait --timeout=90s -n cert-manager endpoints/cert-manager-webhook --for=jsonpath='{.subsets[0].addresses[0].ip}' # Wait for few seconds for the cert-manager API to be ready # otherwise you'll hit the error "x509: certificate signed by unknown authority" # Best is to use cmctl - https://cert-manager.io/docs/installation/kubectl/#2-optional-wait-for-cert-manager-webhook-to-be-ready