Skip to content

Commit

Permalink
WIP: e2e: add initial SGX EPC cgroups tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>
  • Loading branch information
@mythi
mythi committed Feb 8, 2024
1 parent 7fc5228 commit af4e528
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 3 deletions.
1 change: 1 addition & 0 deletions .github/workflows/lib-e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ jobs:
- name: e2e-sgx
runner: sgx
images: intel-sgx-plugin intel-sgx-initcontainer intel-sgx-admissionwebhook sgx-sdk-demo intel-deviceplugin-operator stress-ng-gramine
targetjob: e2e-sgx SKIP=App:sgx-epc-cgroup

name: ${{ matrix.name }}
runs-on: ${{ matrix.runner }}
Expand Down
4 changes: 2 additions & 2 deletions scripts/set-version.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
exit 1
fi

files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)

for file in $files; do
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
done
111 changes: 110 additions & 1 deletion test/e2e/sgx/sgx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package sgx

import (
"context"
"fmt"
"path/filepath"
"time"

Expand All @@ -28,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/kubernetes/test/e2e/framework"
e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
admissionapi "k8s.io/pod-security-admission/api"
Expand All @@ -37,7 +39,8 @@ const (
ns = "inteldeviceplugins-system"
timeout = time.Second * 120
kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
kustomizationPlugin = "deployments/sgx_plugin/base/kustomization.yaml"
kustomizationPlugin = "deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml"
stressNGImage = "intel/stress-ng-gramine:devel"
)

func init() {
Expand Down Expand Up @@ -80,6 +83,9 @@ func describe() {
})

ginkgo.Context("When SGX resources are available", func() {
var nodeWithEPC string
var epcCapacity int64

ginkgo.BeforeEach(func(ctx context.Context) {
ginkgo.By("checking if the resource is allocatable")
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/epc", 150*time.Second); err != nil {
Expand All @@ -91,6 +97,20 @@ func describe() {
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/provision", 30*time.Second); err != nil {
framework.Failf("unable to wait for nodes to have positive allocatable provision resource: %v", err)
}

nodelist, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
framework.Failf("failed to list Nodes: %v", err)
}

// we have at least one node with sgx.intel.com/epc capacity
for _, item := range nodelist.Items {
if q, ok := item.Status.Allocatable["sgx.intel.com/epc"]; ok && q.Value() > 0 {
epcCapacity = q.Value()
nodeWithEPC = item.Name
break
}
}
})

ginkgo.It("deploys a sgx-sdk-demo pod requesting SGX enclave resources [App:sgx-sdk-demo]", func(ctx context.Context) {
Expand Down Expand Up @@ -120,6 +140,95 @@ func describe() {
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, "testcontainer"))
})

ginkgo.It("deploys simultaneous SGX EPC stressor jobs with equal EPC limits but no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
parallelism := int32(20)
completions := int32(20)
quantity := resource.NewQuantity(epcCapacity/int64(parallelism), resource.BinarySI)

testArgs := []string{
"stress-ng-edmm",
"--vm",
"1",
"--vm-bytes",
fmt.Sprintf("%db", epcCapacity/int64(parallelism)),
"--page-in",
"-t",
"30",
}
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, parallelism, completions, nil, 1, nodeWithEPC)

job.Spec.Template.Spec.Containers[0].Image = stressNGImage
job.Spec.Template.Spec.Containers[0].Args = testArgs
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity},
}

job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)

err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, completions)
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
})

ginkgo.It("deploys one SGX EPC stressor job with a known enclave size and no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
quantity := resource.NewQuantity(epcCapacity, resource.BinarySI)

testArgs := []string{
"stress-ng",
"--vm",
"1",
"--vm-bytes",
"128m",
"--page-in",
"-t",
"30",
}
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 1, nodeWithEPC)

job.Spec.Template.Spec.Containers[0].Image = stressNGImage
job.Spec.Template.Spec.Containers[0].Args = testArgs
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity},
}

job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)

err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, 1)
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
})

ginkgo.It("deploys one SGX EPC stressor job with dynamic EPC allocation and memory limit set to kill once enough EPC pages are reclaimed [App:sgx-epc-cgroup]", func(ctx context.Context) {
quantity := resource.NewQuantity(epcCapacity/10, resource.BinarySI)

testArgs := []string{
"stress-ng-edmm",
"--bigheap",
"1",
"--bigheap-growth",
"1M",
"--page-in",
"-t",
"60",
}
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob", v1.RestartPolicyNever, 1, 1, nil, 1, nodeWithEPC)

job.Spec.Template.Spec.Containers[0].Image = stressNGImage
job.Spec.Template.Spec.Containers[0].Args = testArgs
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
v1.ResourceMemory: *quantity},
}

job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
err = e2ejob.WaitForJobFailed(f.ClientSet, f.Namespace.Name, job.Name)
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.It("does nothing", func() {})
})
Expand Down

0 comments on commit af4e528

Please sign in to comment.