kbs: Remove oci.kbs version and references

Our KBS logic currently checks out the git.kbs version of code, but then edits the deployment config to override the image based on the oci.kbs version. This means that there is an assumption that the oci image of the KBS is compatible with the kustomization, which might not not always be the case and in the "always safe" case where the image is built from that exact version of code (e.g. releases), just means that we have to specify it in two places, so there isn't much advantage to this approach. This PR removes the oci.kbs image and versioning to avoid this potential incompatibility and simplify trustee version updates. Because there is a chance that the trustee team might not keep their kustomizational image up-to-date, the recommendation is that we should ensure that the image version matches the branch we are using, so perform a kustomize set image at the time we clone the repo. As trustee have a separate container registry for development versus release images for reasons, they only automatically push builds to the staged-images registry, which are tagged by the sha of the commit, so we need to convert the KBS_VERSION given into the sha and then replace the image with the staged-images kbs Fixes: confidential-containers#2076 Signed-off-by: stevenhorsman <steven@uk.ibm.com>
davidhadas · Oct 23, 2024 · 860a07d · 860a07d
1 parent 68e3f6c
commit 860a07d
Show file tree

Hide file tree

Showing 15 changed files with 49 additions and 107 deletions.
diff --git a/.github/workflows/azure-e2e-test.yml b/.github/workflows/azure-e2e-test.yml
@@ -28,12 +28,6 @@ on:
       caa-image:
         type: string
         description: prebuilt caa image
-      kbs-image-id:
-        type: string
-        description: prebuild kbs image
-      kbs-image-tag:
-        type: string
-        description: prebuild kbs image tag
 
 jobs:
   build-caa-container-image:
@@ -117,17 +111,10 @@ jobs:
           echo "TEST_PROVISION_FILE=${{ format(env.TEST_PROVISION_PATH_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV"
           echo "CLUSTER_NAME=${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV"
 
-    - name: Extract provisioner environment
-      run: |
-        echo "KBS_IMAGE=$(yq -e '.oci.kbs.registry' versions.yaml)" >> "$GITHUB_ENV"
-        echo "KBS_IMAGE_TAG=$(yq -e '.oci.kbs.tag' versions.yaml)" >> "$GITHUB_ENV"
-
     - name: Create provisioner file
       env:
         AZURE_IMAGE_ID: ${{ inputs.podvm-image-id }}
         CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}"
-        KBS_IMAGE: ${{ github.event.inputs.kbs-image-id || env.KBS_IMAGE }}
-        KBS_IMAGE_TAG: ${{  github.event.inputs.kbs-image-tag || env.KBS_IMAGE_TAG}}
         AZURE_INSTANCE_SIZE: ${{ matrix.parameters.machine_type }}
       run: |
         echo "Provisioner file for ${{ matrix.parameters.id }}"
@@ -142,8 +129,6 @@ jobs:
           IS_CI_MANAGED_CLUSTER="true"
           MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}"
           CAA_IMAGE="${CAA_IMAGE}"
-          KBS_IMAGE="${KBS_IMAGE}"
-          KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"
           AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}"
           TAGS="${{ env.TEST_TAGS }}"
           FEDERATED_CREDENTIAL_NAME="${{ env.CLUSTER_NAME }}"
@@ -250,6 +235,14 @@ jobs:
         git clone https://github.com/confidential-containers/trustee test/trustee
         pushd test/trustee
         git checkout "${KBS_VERSION}"
+        pushd kbs/config/kubernetes/base/
+        # Trustee only updates their staging image reliably with sha tags,
+        # so switch to use that and convert the version to the sha
+        KBS_SHA=$(gh api repos/confidential-containers/trustee/commits/${KBS_VERSION} -q .sha)
+        kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:${KBS_SHA}
+        # For debugging
+        echo "Trustee deployment: $(cat kustomization.yaml). Images: $(cat kustomization.yaml | grep -A 5 images:)"
+        popd
         popd
 
     - name: Run e2e test

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
@@ -108,6 +108,14 @@ jobs:
           git checkout "${KBS_VERSION}"
           pushd kbs
           make CLI_FEATURES=sample_only cli
+          pushd config/kubernetes/base/
+          # Trustee only updates their staging image reliably with sha tags,
+          # so switch to use that and convert the version to the sha
+          KBS_SHA=$(gh api repos/confidential-containers/trustee/commits/${KBS_VERSION} -q .sha)
+          kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:${KBS_SHA}
+          # For debugging
+          echo "Trustee deployment: $(cat kustomization.yaml). Images: $(cat kustomization.yaml | grep -A 5 images:)"
+          popd
           popd
           # For debugging
           ls ./target/release

diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh
@@ -118,7 +118,3 @@ rm -f libvirt.properties
 echo "libvirt_uri=\"qemu+ssh://${USER}@${IP}/system?no_verify=1\"" >> libvirt.properties
 echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties
 echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
-KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml)
-KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml)
-[ -z ${KBS_IMAGE} ] || echo "KBS_IMAGE=\"${KBS_IMAGE}\"" >> libvirt.properties
-[ -z ${KBS_IMAGE_TAG} ] || echo "KBS_IMAGE_TAG=\"${KBS_IMAGE_TAG}\"" >> libvirt.properties
diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md
@@ -63,8 +63,15 @@ To prepare trustee, execute the following steps:
 pushd ${cloud-api-adaptor-repo-dir}/src/cloud-api-adaptor/test
 git clone https://github.com/confidential-containers/trustee.git
 pushd trustee
-git checkout $(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml)
+KBS_VERSION=$(../../hack/yq-shim.sh '.git.kbs.reference' ../../versions.yaml)
+git checkout ${KBS_VERSION}
 pushd kbs
+pushd config/kubernetes/base/
+# Trustee only updates their staging image reliably with sha tags,
+# so switch to use that and convert the version to the sha
+KBS_SHA=$(gh api repos/confidential-containers/trustee/commits/${KBS_VERSION} -q .sha)
+kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:${KBS_SHA}
+popd
 make CLI_FEATURES=sample_only cli
 popd
 popd
@@ -81,12 +88,10 @@ popd
 
 Then extract the PodVM image and use it following [extracting-the-qcow2-image](../../podvm/README.md#extracting-the-qcow2-image)
 
-To deploy the KBS service and test attestation related cases, export the following variables like:
+To deploy the KBS service and test attestation related cases, export the following variable:
 
 ```sh
 export DEPLOY_KBS=yes
-export KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml)
-export KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml)
 ````
 
 # Running end-to-end tests against pre-configured cluster
@@ -256,4 +261,4 @@ func TestCloudProviderCreateSimplePod(t *testing.T) {
     assert := MyAssert{}
     DoTestCreateSimplePod(t, assert)
 }
-```
+```
diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go
@@ -5,7 +5,6 @@ package e2e
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"testing"
 
@@ -123,13 +122,7 @@ func TestMain(m *testing.M) {
 		var err error
 
 		// Get properties
-
 		props := provisioner.GetProperties(ctx, cfg)
-		if shouldDeployKbs {
-			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
-				return ctx, fmt.Errorf("kbs image not provided")
-			}
-		}
 
 		// Set CONTAINER_RUNTIME env variable if present in the properties
 		// Default value is containerd.

diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure.properties
@@ -14,10 +14,6 @@ AZURE_CLI_AUTH="false"
 IS_CI_MANAGED_CLUSTER="false"
 IS_SELF_MANAGED_CLUSTER="false"
 
-# KBS configs
-KBS_IMAGE="${KBS_IMAGE}"
-KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"
-
 AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}"
 
 # either "containerd" or "crio"

diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties b/src/cloud-api-adaptor/test/provisioner/azure/provision_azure_cli_auth.properties
@@ -10,6 +10,4 @@ SSH_USERNAME=""
 AZURE_CLI_AUTH="true"
 IS_CI_MANAGED_CLUSTER="true"
 CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor:latest"
-KBS_IMAGE="${KBS_IMAGE}"
-KBS_IMAGE_TAG="${KBS_IMAGE_TAG}"
 AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}"
diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go
@@ -359,8 +359,6 @@ func getPropertiesImpl() map[string]string {
 		"AZURE_IMAGE_ID":        AzureProps.ImageID,
 		"AZURE_SUBNET_ID":       AzureProps.SubnetID,
 		"AZURE_INSTANCE_SIZE":   AzureProps.InstanceSize,
-		"KBS_IMAGE":             AzureProps.KbsImage,
-		"KBS_IMAGE_TAG":         AzureProps.KbsImageTag,
 		"TAGS":                  AzureProps.Tags,
 		"CONTAINER_RUNTIME":     AzureProps.ContainerRuntime,
 	}

diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_initializer.go
@@ -33,8 +33,6 @@ type AzureProperties struct {
 	IsCIManaged         bool
 	CaaImage            string
 	IsSelfManaged       bool
-	KbsImage            string
-	KbsImageTag         string
 	Tags                string
 
 	InstanceSize string
@@ -68,8 +66,6 @@ func initAzureProperties(properties map[string]string) error {
 		SshUserName:             properties["SSH_USERNAME"],
 		ManagedIdentityName:     properties["MANAGED_IDENTITY_NAME"],
 		CaaImage:                properties["CAA_IMAGE"],
-		KbsImage:                properties["KBS_IMAGE"],
-		KbsImageTag:             properties["KBS_IMAGE_TAG"],
 		InstanceSize:            properties["AZURE_INSTANCE_SIZE"],
 		Tags:                    properties["TAGS"],
 		FederatedCredentialName: properties["FEDERATED_CREDENTIAL_NAME"],

diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go
@@ -38,8 +38,6 @@ type DockerProperties struct {
 	PodvmImage       string
 	CaaImage         string
 	CaaImageTag      string
-	KbsImage         string
-	KbsImageTag      string
 	ContainerRuntime string
 }
 
@@ -55,8 +53,6 @@ func initDockerProperties(properties map[string]string) error {
 		PodvmImage:       properties["DOCKER_PODVM_IMAGE"],
 		CaaImage:         properties["CAA_IMAGE"],
 		CaaImageTag:      properties["CAA_IMAGE_TAG"],
-		KbsImage:         properties["KBS_IMAGE"],
-		KbsImageTag:      properties["KBS_IMAGE_TAG"],
 		ContainerRuntime: properties["CONTAINER_RUNTIME"],
 	}
 	return nil
@@ -138,8 +134,6 @@ func (l *DockerProvisioner) GetProperties(ctx context.Context, cfg *envconf.Conf
 		"DOCKER_PODVM_IMAGE":  DockerProps.PodvmImage,
 		"CAA_IMAGE":           DockerProps.CaaImage,
 		"CAA_IMAGE_TAG":       DockerProps.CaaImageTag,
-		"KBS_IMAGE":           DockerProps.KbsImage,
-		"KBS_IMAGE_TAG":       DockerProps.KbsImageTag,
 		"CONTAINER_RUNTIME":   DockerProps.ContainerRuntime,
 	}
 }

diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties
@@ -6,9 +6,5 @@ DOCKER_NETWORK_NAME="kind"
 CAA_IMAGE=""
 CAA_IMAGE_TAG=""
 
-# KBS configs
-KBS_IMAGE=""
-KBS_IMAGE_TAG=""
-
 # either "containerd" or "crio"
 CONTAINER_RUNTIME="containerd"
diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go
@@ -21,16 +21,14 @@ import (
 
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
-	conn          *libvirt.Connect // Libvirt connection
-	network       string           // Network name
-	ssh_key_file  string           // SSH key file used to connect to Libvirt
-	storage       string           // Storage pool name
-	uri           string           // Libvirt URI
-	wd            string           // libvirt's directory path on this repository
-	volumeName    string           // Podvm volume name
-	clusterName   string           // Cluster name
-	kbs_image     string           // KBS Service OCI Image URL
-	kbs_image_tag string           // KBS Service OCI Image Tag
+	conn         *libvirt.Connect // Libvirt connection
+	network      string           // Network name
+	ssh_key_file string           // SSH key file used to connect to Libvirt
+	storage      string           // Storage pool name
+	uri          string           // Libvirt URI
+	wd           string           // libvirt's directory path on this repository
+	volumeName   string           // Podvm volume name
+	clusterName  string           // Cluster name
 }
 
 // LibvirtInstallOverlay implements the InstallOverlay interface
@@ -82,28 +80,16 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 		clusterName = properties["cluster_name"]
 	}
 
-	kbs_image := "ghcr.io/confidential-containers/key-broker-service"
-	if properties["KBS_IMAGE"] != "" {
-		kbs_image = properties["KBS_IMAGE"]
-	}
-
-	kbs_image_tag := "latest"
-	if properties["KBS_IMAGE_TAG"] != "" {
-		kbs_image_tag = properties["KBS_IMAGE_TAG"]
-	}
-
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
-		conn:          conn,
-		network:       network,
-		ssh_key_file:  ssh_key_file,
-		storage:       storage,
-		uri:           uri,
-		wd:            wd,
-		volumeName:    vol_name,
-		clusterName:   clusterName,
-		kbs_image:     kbs_image,
-		kbs_image_tag: kbs_image_tag,
+		conn:         conn,
+		network:      network,
+		ssh_key_file: ssh_key_file,
+		storage:      storage,
+		uri:          uri,
+		wd:           wd,
+		volumeName:   vol_name,
+		clusterName:  clusterName,
 	}, nil
 }
 
@@ -203,13 +189,11 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)
 
 func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
 	return map[string]string{
-		"network":       l.network,
-		"podvm_volume":  l.volumeName,
-		"ssh_key_file":  l.ssh_key_file,
-		"storage":       l.storage,
-		"uri":           l.uri,
-		"KBS_IMAGE":     l.kbs_image,
-		"KBS_IMAGE_TAG": l.kbs_image_tag,
+		"network":      l.network,
+		"podvm_volume": l.volumeName,
+		"ssh_key_file": l.ssh_key_file,
+		"storage":      l.storage,
+		"uri":          l.uri,
 	}
 }
 

diff --git a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go
@@ -346,17 +346,7 @@ func (lio *KbsInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) e
 }
 
 func (lio *KbsInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, props map[string]string) error {
-	var err error
-	log.Infof("Updating kbs image with %q", props["KBS_IMAGE"])
-	if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newName", props["KBS_IMAGE"]); err != nil {
-		return err
-	}
-
-	log.Infof("Updating kbs image tag with %q", props["KBS_IMAGE_TAG"])
-	if err = lio.overlay.SetKustomizeImage("kbs-container-image", "newTag", props["KBS_IMAGE_TAG"]); err != nil {
-		return err
-	}
-
+	// No edits needed here at the moment
 	return nil
 }
 

diff --git a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go
@@ -91,9 +91,7 @@ func main() {
 
 		if shouldDeployKbs {
 			log.Info("Deploying kbs")
-			if props["KBS_IMAGE"] == "" || props["KBS_IMAGE_TAG"] == "" {
-				log.Fatal("kbs image not provided")
-			}
+
 			keyBrokerService, err := pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg)
 			if err != nil {
 				log.Fatal(err)

diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml
@@ -54,6 +54,3 @@ oci:
   pause:
     registry: docker://registry.k8s.io/pause
     tag: 3.9
-  kbs:
-    registry: ghcr.io/confidential-containers/key-broker-service
-    tag: built-in-as-v0.10.1