feat: Confidential compute changes - 2022-10-01-preview

e2e/confidential_container_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the Apache 2.0 license.
+*/
+package e2e
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/virtual-kubelet/azure-aci/pkg/featureflag"
+)
+
+func TestPodWithInitConfidentialContainer(t *testing.T) {
+	ctx := context.TODO()
+	enabledFeatures := featureflag.InitFeatureFlag(ctx)
+	if !enabledFeatures.IsEnabled(ctx, featureflag.ConfidentialComputeFeature) {
+		t.Skipf("%s feature is not enabled", featureflag.ConfidentialComputeFeature)
+	}
+
+	// delete the namespace first
+	cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	// create namespace
+	cmd = kubectl("apply", "-f", "fixtures/namespace.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	// create confidential container pod
+	cmd = kubectl("apply", "-f", "fixtures/confidential_container_pod.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	deadline, ok := t.Deadline()
+	timeout := time.Until(deadline)
+	if !ok {
+		timeout = 300 * time.Second
+	}
+	cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/confidential-container-sevsnp", "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	t.Log("success create pod")
+
+	// query metrics
+	deadline = time.Now().Add(10 * time.Minute)
+	for {
+		t.Log("query metrics ....")
+		cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/confidential-container-sevsnp")
+		out, err := cmd.CombinedOutput()
+		if time.Now().After(deadline) {
+			t.Fatal("failed to query pod's stats from metrics server API")
+		}
+		if err == nil {
+			t.Logf("success query metrics %s", string(out))
+			break
+		}
+		time.Sleep(10 * time.Second)
+	}
+
+	// check pod status
+	t.Log("get pod status ....")
+	cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "confidential-container-sevsnp" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query pod status %s", string(out))
+
+	// check container status
+	t.Log("get container status ....")
+	cmd = kubectl("get", "pod", "confidential-container-sevsnp", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Fatal(string(out))
+	}
+	if string(out) != "true" {
+		t.Fatal("failed to get pod's status")
+	}
+	t.Logf("success query container status %s", string(out))
+
+	t.Log("clean up pod")
+	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+}

e2e/fixtures/confidential_container_pod.yml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: confidential-container-sevsnp
+  namespace: vk-test
+  annotations:
+    virtual-kubelet.io/container-sku: "Confidential"
+spec:
+  restartPolicy: Always
+  containers:
+  - image: docker.io/kengordon/parmasimple:latest
+    imagePullPolicy: Always
+    name: e2etest-conf-container
+    resources:
+      requests:
+        memory: 1G
+        cpu: 1
+    ports:
+    - containerPort: 8000
+      name: http
+  nodeSelector:
+    kubernetes.io/role: agent
+    beta.kubernetes.io/os: linux
+    type: virtual-kubelet
+  tolerations:
+  - key: virtual-kubelet.io/provider
+    operator: Exists

hack/e2e/aks-addon.sh

@@ -20,7 +20,7 @@ if [ "$PR_RAND" = "" ]; then
 fi
 
 : "${RESOURCE_GROUP:=aks-addon-aci-test-$RANDOM_NUM}"
-: "${LOCATION:=eastus2}"
+: "${LOCATION:=eastus2euap}"
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=aks-addon--test}"

hack/e2e/aks.sh

@@ -20,7 +20,7 @@ if [ "$PR_RAND" = "" ]; then
 fi
 
 : "${RESOURCE_GROUP:=vk-aci-test-$RANDOM_NUM}"
-: "${LOCATION:=eastus2}"
+: "${LOCATION:=eastus2euap}"
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=vk-aci-test-aks}"

pkg/featureflag/feature_flag.go

@@ -12,9 +12,13 @@ import (
 
 const (
 	InitContainerFeature = "init-container"
+	ConfidentialComputeFeature = "confidential-compute"
 )
 
-var enabledFeatures = []string{}
+var enabledFeatures = []string{
+	InitContainerFeature,
+	ConfidentialComputeFeature,
+}
 
 type FlagIdentifier struct {
 	enabledFeatures []string

pkg/featureflag/feature_flag_test.go

@@ -16,9 +16,14 @@ func TestIsEnabled(t *testing.T) {
 		shouldEnabled bool
 	}{
 		{
-			description:   fmt.Sprintf(" %s feature should not be enabled", InitContainerFeature),
+			description:   fmt.Sprintf(" %s feature should be enabled", InitContainerFeature),
 			feature:       InitContainerFeature,
-			shouldEnabled: false,
+			shouldEnabled: true,
+		},
+		{
+			description:   fmt.Sprintf(" %s feature should be enabled", ConfidentialComputeFeature),
+			feature:       ConfidentialComputeFeature,
+			shouldEnabled: true,
 		},
 	}
 	for i, tc := range cases {

pkg/provider/aci.go

@@ -63,6 +63,11 @@ const (
 	containerExitCodePodDeleted int32 = 0
 )
 
+const (
+	confidentialComputeSkuLabel = "virtual-kubelet.io/container-sku"
+	confidentialComputeCcePolicyLabel = "virtual-kubelet.io/confidential-compute-cce-policy"
+)
+
 // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs.
 type ACIProvider struct {
 	azClientsAPIs            client.AzClientsInterface
@@ -317,6 +322,12 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 		cg.Properties.InitContainers = initContainers
 	}
 
+	// confidential compute proeprties
+	if p.enabledFeatures.IsEnabled(ctx, featureflag.ConfidentialComputeFeature) {
+		// set confidentialComputeProperties
+		p.setConfidentialComputeProperties(ctx, pod, cg)
+	}
+
 	// assign all the things
 	cg.Properties.Containers = containers
 	cg.Properties.Volumes = volumes
@@ -1114,6 +1125,29 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) ([]*azaciv2.Container, error) {
 	return containers, nil
 }
 
+func (p *ACIProvider) setConfidentialComputeProperties(ctx context.Context, pod *v1.Pod, cg *azaciv2.ContainerGroup) {
+	containerGroupSku := pod.Annotations[confidentialComputeSkuLabel]
+	ccePolicy := pod.Annotations[confidentialComputeCcePolicyLabel]
+	confidentialSku := azaciv2.ContainerGroupSKUConfidential
+
+	l := log.G(ctx).WithField("containerGroup", cg.Name)
+
+	if ccePolicy != "" {
+		cg.Properties.SKU = &confidentialSku
+		confidentialComputeProperties := azaciv2.ConfidentialComputeProperties{
+			CcePolicy : &ccePolicy,
+		}
+		cg.Properties.ConfidentialComputeProperties = &confidentialComputeProperties
+	    l.Infof("setting confidential compute properties with CCE Policy")
+
+	} else if strings.ToLower(containerGroupSku) == "confidential" {
+		cg.Properties.SKU = &confidentialSku
+	    l.Infof("setting confidential container group SKU")
+	}
+
+	l.Infof("no annotations for confidential SKU")
+}
+
 func (p *ACIProvider) getGPUSKU(pod *v1.Pod) (azaciv2.GpuSKU, error) {
 	if len(p.gpuSKUs) == 0 {
 		return "", fmt.Errorf("the pod requires GPU resource, but ACI doesn't provide GPU enabled container group in region %s", p.region)

pkg/provider/aci_confidential_test.go

@@ -0,0 +1,156 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the Apache 2.0 license.
+*/
+package provider
+
+import (
+	"context"
+	"testing"
+
+	azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2"
+	"github.com/golang/mock/gomock"
+	"github.com/virtual-kubelet/azure-aci/pkg/featureflag"
+	"github.com/virtual-kubelet/node-cli/manager"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestCreatePodWithConfidentialComputeProperties(t *testing.T) {
+
+	initContainerName1 := "init-container-1"
+	ccePolicyString	   := "eyJhbGxvd19hbGwiOiB0cnVlLCAiY29udGFpbmVycyI6IHsibGVuZ3RoIjogMCwgImVsZW1lbnRzIjogbnVsbH19"
+	mockCtrl           := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	aciMocks := createNewACIMock()
+	aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *azaciv2.ContainerGroup) error {
+		containers := cg.Properties.Containers
+		initContainers := cg.Properties.InitContainers
+		confidentialComputeProperties := cg.Properties.ConfidentialComputeProperties
+		assert.Check(t, cg != nil, "Container group is nil")
+		assert.Check(t, containers != nil, "Containers should not be nil")
+		assert.Check(t, initContainers != nil, "Container group is nil")
+		if (len(initContainers) > 0) {
+			assert.Check(t, is.Equal(len(containers), 2), "2 Containers are expected")
+			assert.Check(t, is.Equal(len(initContainers), 1), "2 init containers are expected")
+			assert.Check(t, initContainers[0].Properties.VolumeMounts != nil, "Volume mount should be present")
+			assert.Check(t, initContainers[0].Properties.EnvironmentVariables != nil, "Volume mount should be present")
+			assert.Check(t, initContainers[0].Properties.Command != nil, "Command mount should be present")
+			assert.Check(t, initContainers[0].Properties.Image != nil, "Image should be present")
+			assert.Check(t, *initContainers[0].Name == initContainerName1, "Name should be correct")
+		}
+		if (confidentialComputeProperties != nil) {
+			assert.Check(t, confidentialComputeProperties.CcePolicy != nil, "CCE policy should not be nil")
+			assert.Check(t, *confidentialComputeProperties.CcePolicy == ccePolicyString, "CCE policy should match")
+		}
+		assert.Check(t, *cg.Properties.SKU == azaciv2.ContainerGroupSKUConfidential, "Container group sku should be confidential")
+
+		return nil
+	}
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: podNamespace,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "container-name-01",
+					Image: "alpine",
+				},
+				{
+					Name:  "container-name-02",
+					Image: "alpine",
+				},
+			},
+		},
+	}
+	cases := []struct {
+		description    string
+		initContainers []v1.Container
+		annotations    map[string]string
+		expectedError  error
+	}{
+		{
+			description:   "create confidential container group with wildcard policy",
+			expectedError: nil,
+			annotations: map[string]string{
+				confidentialComputeSkuLabel: "Confidential",
+			},
+			initContainers: nil,
+		},
+		{
+			description:   "create confidential container group with specified cce policy",
+			expectedError: nil,
+			annotations: map[string]string{
+				confidentialComputeCcePolicyLabel: ccePolicyString,
+			},
+			initContainers: nil,
+		},
+		{
+			description:   "create confidential container group with init container",
+			expectedError: nil,
+			annotations: map[string]string{
+				confidentialComputeSkuLabel: "Confidential",
+			},
+			initContainers: []v1.Container{
+				v1.Container{
+					Name:  initContainerName1,
+					Image: "alpine",
+					VolumeMounts: []v1.VolumeMount{
+						v1.VolumeMount{
+							Name:      "fakeVolumeName",
+							MountPath: "/mnt/azure",
+						},
+					},
+					Command: []string{"/bin/bash"},
+					Args:    []string{"-c echo test"},
+					Env: []v1.EnvVar{
+						v1.EnvVar{
+							Name:  "TEST_ENV",
+							Value: "testvalue",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.description, func(t *testing.T) {
+
+			ctx := context.TODO()
+
+			resourceManager, err := manager.NewResourceManager(
+				NewMockPodLister(mockCtrl),
+				NewMockSecretLister(mockCtrl),
+				NewMockConfigMapLister(mockCtrl),
+				NewMockServiceLister(mockCtrl),
+				NewMockPersistentVolumeClaimLister(mockCtrl),
+				NewMockPersistentVolumeLister(mockCtrl))
+			if err != nil {
+				t.Fatal("Unable to prepare the mocks for resourceManager", err)
+			}
+
+			provider, err := createTestProvider(aciMocks, resourceManager)
+			if err != nil {
+				t.Fatal("Unable to create test provider", err)
+			}
+
+			if !provider.enabledFeatures.IsEnabled(ctx, featureflag.InitContainerFeature) {
+				t.Skipf("%s feature is not enabled", featureflag.InitContainerFeature)
+			}
+
+			pod.Annotations = tc.annotations
+			pod.Spec.InitContainers = tc.initContainers
+			err = provider.CreatePod(context.Background(), pod)
+
+			// check that the correct error is returned
+			if tc.expectedError != nil && err != tc.expectedError {
+				assert.Equal(t, tc.expectedError.Error(), err.Error(), "expected error and actual error don't match")
+			}
+		})
+	}
+}