From 0fa251f3552dc2cc642fc7b69addc1a2b512deb6 Mon Sep 17 00:00:00 2001 From: Konstantinos Angelopoulos Date: Tue, 26 Mar 2024 09:55:16 +0100 Subject: [PATCH 1/5] confidential vm support --- .../machineclass/templates/machineclass.yaml | 7 +++++ pkg/controller/worker/machines.go | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/charts/internal/machineclass/templates/machineclass.yaml b/charts/internal/machineclass/templates/machineclass.yaml index f24888a60..0a1592832 100644 --- a/charts/internal/machineclass/templates/machineclass.yaml +++ b/charts/internal/machineclass/templates/machineclass.yaml @@ -36,6 +36,10 @@ nodeTemplate: providerSpec: location: {{ $machineClass.region }} properties: + {{- if $machineClass.securityProfile }} + securityProfile: + securityType: {{ $machineClass.securityProfile.securityType }} + {{- end }} {{- if hasKey $machineClass "zone" }} zone: {{ $machineClass.zone }} {{- end }} @@ -71,6 +75,9 @@ providerSpec: sharedGalleryImageID: {{ $machineClass.image.sharedGalleryImageID }} {{- else }} urn: {{ $machineClass.image.urn }} +{{- end }} +{{- if $machineClass.image.privatePlan }} + privatePlan: {{ $machineClass.image.privatePlan }} {{- end }} osDisk: caching: None diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 09200ab1f..23b15bfd0 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -12,6 +12,7 @@ import ( "sort" "strings" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/gardener/gardener/extensions/pkg/controller/worker" genericworkeractuator "github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -20,6 +21,7 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/utils" machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -85,6 +87,7 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { machineDeployments = worker.MachineDeployments{} machineClasses []map[string]interface{} machineImages []azureapi.MachineImage + skipAgreementPools = sets.New[string]() ) infrastructureStatus, err := w.decodeAzureInfrastructureStatus() @@ -102,6 +105,12 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { return err } + if v, ok := w.cluster.Shoot.GetAnnotations()["azure.provider.extensions.gardener.cloud/skip-marketplace-agreement"]; ok { + for _, p := range strings.Split(v, ",") { + skipAgreementPools.Insert(p) + } + } + for _, pool := range w.worker.Spec.Pools { // Get the vmo dependency from the worker status if exists. vmoDependency, err := w.determineWorkerPoolVmoDependency(ctx, infrastructureStatus, workerStatus, pool.Name) @@ -136,6 +145,9 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } else { image["id"] = *id } + if skipAgreementPools.Has(pool.Name) { + image["privatePlan"] = true + } disks, err := computeDisks(pool) if err != nil { @@ -267,6 +279,13 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } } + // special processing of confidential VMs. + if w.isConfidentialVM(pool.MachineType) { + machineClassSpec["securityProfile"] = map[string]interface{}{ + "securityType": string(armcompute.SecurityTypesConfidentialVM), + } + } + machineDeployment.ClusterAutoscalerAnnotations = extensionsv1alpha1helper.GetMachineDeploymentClusterAutoscalerAnnotations(pool.ClusterAutoscaler) return machineDeployment, machineClassSpec @@ -471,3 +490,11 @@ func (w *workerDelegate) generateWorkerPoolHash(pool extensionsv1alpha1.WorkerPo } return workerPoolHash, nil } + +// TODO(AK): Remove when we have support for VM Capabilities +func (w *workerDelegate) isConfidentialVM(family string) bool { + if strings.HasPrefix(strings.ToLower(family), "standard_ec") || strings.HasPrefix(strings.ToLower(family), "standard_dc") { + return true + } + return false +} From 98464630330a6667d8de0d40887ca82a1895d639 Mon Sep 17 00:00:00 2001 From: Konstantinos Angelopoulos Date: Fri, 3 May 2024 08:40:21 +0200 Subject: [PATCH 2/5] use annotations for skipping private plan --- .../machineclass/templates/machineclass.yaml | 4 ++++ pkg/azure/types.go | 9 +++++++++ pkg/controller/worker/machines.go | 16 +++++++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/charts/internal/machineclass/templates/machineclass.yaml b/charts/internal/machineclass/templates/machineclass.yaml index 0a1592832..1a5134c2e 100644 --- a/charts/internal/machineclass/templates/machineclass.yaml +++ b/charts/internal/machineclass/templates/machineclass.yaml @@ -18,6 +18,10 @@ kind: MachineClass metadata: name: {{ $machineClass.name }} namespace: {{ $.Release.Namespace }} + annotations: + {{- if $machineClass.annotations }} +{{ toYaml $machineClass.annotations | indent 4 }} + {{- end }} labels: {{- if $machineClass.operatingSystem }} {{ toYaml $machineClass.operatingSystem | indent 4 }} diff --git a/pkg/azure/types.go b/pkg/azure/types.go index 3bdec7c63..17f12c3b1 100644 --- a/pkg/azure/types.go +++ b/pkg/azure/types.go @@ -141,10 +141,19 @@ const ( SeedAnnotationKeyUseFlow = AnnotationKeyUseFlow // SeedAnnotationUseFlowValueNew is the value to restrict flow reconciliation to new shoot clusters SeedAnnotationUseFlowValueNew = "new" + + // BetaSkipMarketPlaceAgreementAnnotation when present will instruct the worker controller to create machine classes that skip the marketplace agreement by MCM. + BetaSkipMarketPlaceAgreementAnnotation = "beta.azure.provider.extensions.gardener.cloud/skip-marketplace-agreement" ) // UsernamePrefix is a constant for the username prefix of components deployed by Azure. var ( UsernamePrefix = extensionsv1alpha1.SchemeGroupVersion.Group + ":" + Name + ":" ValidFlowAnnotations = []string{AnnotationKeyUseFlow, GlobalAnnotationKeyUseFlow} + + // ConfidentialVMFamilyPrefixes is a list of known families that are used for confidential VMs. + ConfidentialVMFamilyPrefixes = []string{ + "standard_ec", + "standard_dc", + } ) diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 23b15bfd0..0661190b0 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -28,6 +28,7 @@ import ( "github.com/gardener/gardener-extension-provider-azure/charts" azureapi "github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure" azureapihelper "github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure/helper" + "github.com/gardener/gardener-extension-provider-azure/pkg/azure" ) const azureCSIDiskDriverTopologyKey = "topology.disk.csi.azure.com/zone" @@ -105,7 +106,7 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { return err } - if v, ok := w.cluster.Shoot.GetAnnotations()["azure.provider.extensions.gardener.cloud/skip-marketplace-agreement"]; ok { + if v, ok := w.cluster.Shoot.GetAnnotations()[azure.BetaSkipMarketPlaceAgreementAnnotation]; ok { for _, p := range strings.Split(v, ",") { skipAgreementPools.Insert(p) } @@ -145,8 +146,10 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } else { image["id"] = *id } + + machineClassAnnotations := map[string]string{} if skipAgreementPools.Has(pool.Name) { - image["privatePlan"] = true + machineClassAnnotations[azure.BetaSkipMarketPlaceAgreementMCMAnnotation] = "true" } disks, err := computeDisks(pool) @@ -180,6 +183,7 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } machineClassSpec = utils.MergeMaps(map[string]interface{}{ + "annotations": machineClassAnnotations, "region": w.worker.Spec.Region, "resourceGroup": infrastructureStatus.ResourceGroup.Name, "tags": w.getVMTags(pool), @@ -491,10 +495,12 @@ func (w *workerDelegate) generateWorkerPoolHash(pool extensionsv1alpha1.WorkerPo return workerPoolHash, nil } -// TODO(AK): Remove when we have support for VM Capabilities +// TODO: Remove when we have support for VM Capabilities func (w *workerDelegate) isConfidentialVM(family string) bool { - if strings.HasPrefix(strings.ToLower(family), "standard_ec") || strings.HasPrefix(strings.ToLower(family), "standard_dc") { - return true + for _, v := range azure.ConfidentialVMFamilyPrefixes { + if strings.HasPrefix(strings.ToLower(family), strings.ToLower(v)) { + return true + } } return false } From 3d2df04a887880474bf6f06e26ff37073dee1123 Mon Sep 17 00:00:00 2001 From: Konstantinos Angelopoulos Date: Fri, 3 May 2024 14:49:52 +0200 Subject: [PATCH 3/5] update tests --- pkg/controller/worker/machines_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/worker/machines_test.go b/pkg/controller/worker/machines_test.go index 9e2f8c909..f2154323c 100644 --- a/pkg/controller/worker/machines_test.go +++ b/pkg/controller/worker/machines_test.go @@ -427,6 +427,7 @@ var _ = Describe("Machines", func() { } defaultMachineClass := map[string]interface{}{ + "annotations": map[string]string{}, "region": region, "resourceGroup": resourceGroupName, "network": map[string]interface{}{ From 580f336ed46462e9b7384b7ef42982de198ec3a6 Mon Sep 17 00:00:00 2001 From: Konstantinos Angelopoulos Date: Wed, 8 May 2024 09:20:05 +0200 Subject: [PATCH 4/5] v2 --- .../machineclass/templates/machineclass.yaml | 22 ++++++++++------ charts/internal/machineclass/values.yaml | 5 ++++ pkg/controller/worker/machines.go | 26 ++++++++++++------- pkg/controller/worker/machines_test.go | 1 - 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/charts/internal/machineclass/templates/machineclass.yaml b/charts/internal/machineclass/templates/machineclass.yaml index 1a5134c2e..fd1665c31 100644 --- a/charts/internal/machineclass/templates/machineclass.yaml +++ b/charts/internal/machineclass/templates/machineclass.yaml @@ -18,10 +18,6 @@ kind: MachineClass metadata: name: {{ $machineClass.name }} namespace: {{ $.Release.Namespace }} - annotations: - {{- if $machineClass.annotations }} -{{ toYaml $machineClass.annotations | indent 4 }} - {{- end }} labels: {{- if $machineClass.operatingSystem }} {{ toYaml $machineClass.operatingSystem | indent 4 }} @@ -42,7 +38,13 @@ providerSpec: properties: {{- if $machineClass.securityProfile }} securityProfile: + {{- if hasKey $machineClass.securityProfile "securityType" }} securityType: {{ $machineClass.securityProfile.securityType }} + {{- end }} + {{- if hasKey $machineClass.securityProfile "uefiSettings" }} + uefiSettings: +{{ toYaml $machineClass.securityProfile.uefiSettings | indent 8 }} + {{- end }} {{- end }} {{- if hasKey $machineClass "zone" }} zone: {{ $machineClass.zone }} @@ -79,17 +81,21 @@ providerSpec: sharedGalleryImageID: {{ $machineClass.image.sharedGalleryImageID }} {{- else }} urn: {{ $machineClass.image.urn }} -{{- end }} -{{- if $machineClass.image.privatePlan }} - privatePlan: {{ $machineClass.image.privatePlan }} + {{- if $machineClass.image.skipMarketplaceAgreement}} + skipMarketPlaceAgreement: true + {{- end }} {{- end }} osDisk: caching: None diskSizeGB: {{ $machineClass.osDisk.size }} - {{- if hasKey $machineClass.osDisk "type" }} managedDisk: + {{- if hasKey $machineClass.osDisk "type" }} storageAccountType: {{ $machineClass.osDisk.type }} {{- end }} + {{- if hasKey $machineClass.osDisk "securityProfile" }} + securityProfile: + securityEncryptionType: {{ $machineClass.osDisk.securityProfile.securityEncryptionType }} + {{- end }} createOption: FromImage {{- if $machineClass.dataDisks }} dataDisks: diff --git a/charts/internal/machineclass/values.yaml b/charts/internal/machineclass/values.yaml index 546e574ab..53f191df3 100644 --- a/charts/internal/machineclass/values.yaml +++ b/charts/internal/machineclass/values.yaml @@ -31,12 +31,17 @@ machineClasses: zone: westeurope-1 image: urn: "CoreOS:CoreOS:Stable:1576.5.0" + #skipMarketplaceAgreement: true #id: "/subscriptions//resourceGroups/myGalleryRG/providers/Microsoft.Compute/galleries/myGallery/images/myImageDefinition/versions/1.0.0" #communityGalleryImageID: "/CommunityGalleries//Images/myImageDefinition/versions/1.0.0" #sharedGalleryImageID: "/SharedGalleries//Images//Versions/" osDisk: size: 50 #type: Standard_LRS + #securityProfile: + #securityEncryptionType: VMGuestStateOnly + #uefiSettings: + #vtpmEnabled: false sshPublicKey: ssh-rsa AAAAB3... - name: class-2-availability-set region: westeurope diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 0661190b0..726d092e5 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -139,6 +139,9 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { image := map[string]interface{}{} if urn != nil { image["urn"] = *urn + if skipAgreementPools.Has(pool.Name) { + image["skipMarketplaceAgreement"] = true + } } else if communityGalleryImageID != nil { image["communityGalleryImageID"] = *communityGalleryImageID } else if sharedGalleryImageID != nil { @@ -147,11 +150,6 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { image["id"] = *id } - machineClassAnnotations := map[string]string{} - if skipAgreementPools.Has(pool.Name) { - machineClassAnnotations[azure.BetaSkipMarketPlaceAgreementMCMAnnotation] = "true" - } - disks, err := computeDisks(pool) if err != nil { return err @@ -183,7 +181,6 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } machineClassSpec = utils.MergeMaps(map[string]interface{}{ - "annotations": machineClassAnnotations, "region": w.worker.Spec.Region, "resourceGroup": infrastructureStatus.ResourceGroup.Name, "tags": w.getVMTags(pool), @@ -283,10 +280,13 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { } } - // special processing of confidential VMs. - if w.isConfidentialVM(pool.MachineType) { + // special processing of CVMs. + if isConfidentialVM(pool) { machineClassSpec["securityProfile"] = map[string]interface{}{ "securityType": string(armcompute.SecurityTypesConfidentialVM), + "uefiSettings": map[string]interface{}{ + "vtpmEnabled": true, + }, } } @@ -412,6 +412,12 @@ func computeDisks(pool extensionsv1alpha1.WorkerPool) (map[string]interface{}, e osDisk["type"] = *pool.Volume.Type } + if isConfidentialVM(pool) { + osDisk["securityProfile"] = map[string]interface{}{ + "securityEncryptionType": string(armcompute.SecurityEncryptionTypesVMGuestStateOnly), + } + } + disks := map[string]interface{}{ "osDisk": osDisk, } @@ -496,9 +502,9 @@ func (w *workerDelegate) generateWorkerPoolHash(pool extensionsv1alpha1.WorkerPo } // TODO: Remove when we have support for VM Capabilities -func (w *workerDelegate) isConfidentialVM(family string) bool { +func isConfidentialVM(pool extensionsv1alpha1.WorkerPool) bool { for _, v := range azure.ConfidentialVMFamilyPrefixes { - if strings.HasPrefix(strings.ToLower(family), strings.ToLower(v)) { + if strings.HasPrefix(strings.ToLower(pool.MachineType), strings.ToLower(v)) { return true } } diff --git a/pkg/controller/worker/machines_test.go b/pkg/controller/worker/machines_test.go index f2154323c..9e2f8c909 100644 --- a/pkg/controller/worker/machines_test.go +++ b/pkg/controller/worker/machines_test.go @@ -427,7 +427,6 @@ var _ = Describe("Machines", func() { } defaultMachineClass := map[string]interface{}{ - "annotations": map[string]string{}, "region": region, "resourceGroup": resourceGroupName, "network": map[string]interface{}{ From 1e37da163dd21e4d4b2a046c79b9c14d23d0dd1f Mon Sep 17 00:00:00 2001 From: Konstantinos Angelopoulos Date: Tue, 28 May 2024 23:53:55 +0200 Subject: [PATCH 5/5] cloudprofile change --- hack/api-reference/api.md | 24 ++++++++++ pkg/apis/azure/helper/helper.go | 17 +++---- pkg/apis/azure/types_cloudprofile.go | 2 + pkg/apis/azure/types_worker.go | 2 + pkg/apis/azure/v1alpha1/types_cloudprofile.go | 3 ++ pkg/apis/azure/v1alpha1/types_worker.go | 3 ++ .../azure/v1alpha1/zz_generated.conversion.go | 4 ++ .../azure/v1alpha1/zz_generated.deepcopy.go | 10 ++++ pkg/apis/azure/zz_generated.deepcopy.go | 10 ++++ pkg/azure/types.go | 3 -- pkg/controller/worker/machine_images.go | 12 ++--- pkg/controller/worker/machines.go | 47 ++++++++----------- 12 files changed, 93 insertions(+), 44 deletions(-) diff --git a/hack/api-reference/api.md b/hack/api-reference/api.md index a57383d28..a0b0d9239 100644 --- a/hack/api-reference/api.md +++ b/hack/api-reference/api.md @@ -1056,6 +1056,18 @@ string

Architecture is the CPU architecture of the machine image.

+ + +skipMarketplaceAgreement
+ +bool + + + +(Optional) +

SkipMarketplaceAgreement skips the marketplace agreement check when enabled.

+ +

MachineImageVersion @@ -1100,6 +1112,18 @@ string +skipMarketplaceAgreement
+ +bool + + + +(Optional) +

SkipMarketplaceAgreement skips the marketplace agreement check when enabled.

+ + + + id
string diff --git a/pkg/apis/azure/helper/helper.go b/pkg/apis/azure/helper/helper.go index 7786e71cb..33d91e039 100644 --- a/pkg/apis/azure/helper/helper.go +++ b/pkg/apis/azure/helper/helper.go @@ -105,14 +105,15 @@ func FindImageFromCloudProfile(cloudProfileConfig *api.CloudProfileConfig, image for _, version := range machineImage.Versions { if imageVersion == version.Version && ptr.Equal(architecture, version.Architecture) { return &api.MachineImage{ - Name: imageName, - Version: version.Version, - URN: version.URN, - ID: version.ID, - SharedGalleryImageID: version.SharedGalleryImageID, - CommunityGalleryImageID: version.CommunityGalleryImageID, - AcceleratedNetworking: version.AcceleratedNetworking, - Architecture: version.Architecture, + Name: imageName, + Version: version.Version, + URN: version.URN, + ID: version.ID, + SharedGalleryImageID: version.SharedGalleryImageID, + CommunityGalleryImageID: version.CommunityGalleryImageID, + AcceleratedNetworking: version.AcceleratedNetworking, + Architecture: version.Architecture, + SkipMarketplaceAgreement: version.SkipMarketplaceAgreement, }, nil } } diff --git a/pkg/apis/azure/types_cloudprofile.go b/pkg/apis/azure/types_cloudprofile.go index 85a2c4289..936d94d09 100644 --- a/pkg/apis/azure/types_cloudprofile.go +++ b/pkg/apis/azure/types_cloudprofile.go @@ -56,6 +56,8 @@ type MachineImageVersion struct { Version string // URN is the uniform resource name of the image, it has the format 'publisher:offer:sku:version'. URN *string + // SkipMarketplaceAgreement skips the marketplace agreement check when enabled. + SkipMarketplaceAgreement *bool // ID is the Shared Image Gallery image id. ID *string // CommunityGalleryImageID is the Community Image Gallery image id, it has the format '/CommunityGalleries/myGallery/Images/myImage/Versions/myVersion' diff --git a/pkg/apis/azure/types_worker.go b/pkg/apis/azure/types_worker.go index e14477e7b..49498d7a3 100644 --- a/pkg/apis/azure/types_worker.go +++ b/pkg/apis/azure/types_worker.go @@ -54,6 +54,8 @@ type MachineImage struct { AcceleratedNetworking *bool // Architecture is the CPU architecture of the machine image. Architecture *string + // SkipMarketplaceAgreement skips the marketplace agreement check when enabled. + SkipMarketplaceAgreement *bool } // VmoDependency is dependency reference for a workerpool to a VirtualMachineScaleSet Orchestration Mode VM (VMO). diff --git a/pkg/apis/azure/v1alpha1/types_cloudprofile.go b/pkg/apis/azure/v1alpha1/types_cloudprofile.go index 21140fd53..f151a835d 100644 --- a/pkg/apis/azure/v1alpha1/types_cloudprofile.go +++ b/pkg/apis/azure/v1alpha1/types_cloudprofile.go @@ -60,6 +60,9 @@ type MachineImageVersion struct { // URN is the uniform resource name of the image, it has the format 'publisher:offer:sku:version'. // +optional URN *string `json:"urn,omitempty"` + // SkipMarketplaceAgreement skips the marketplace agreement check when enabled. + // +optional + SkipMarketplaceAgreement *bool `json:"skipMarketplaceAgreement,omitempty"` // ID is the Shared Image Gallery image id. // +optional ID *string `json:"id,omitempty"` diff --git a/pkg/apis/azure/v1alpha1/types_worker.go b/pkg/apis/azure/v1alpha1/types_worker.go index e4beac203..27cff8fd0 100644 --- a/pkg/apis/azure/v1alpha1/types_worker.go +++ b/pkg/apis/azure/v1alpha1/types_worker.go @@ -65,6 +65,9 @@ type MachineImage struct { // Architecture is the CPU architecture of the machine image. // +optional Architecture *string `json:"architecture,omitempty"` + // SkipMarketplaceAgreement skips the marketplace agreement check when enabled. + // +optional + SkipMarketplaceAgreement *bool `json:"skipMarketplaceAgreement,omitempty"` } // VmoDependency is dependency reference for a workerpool to a VirtualMachineScaleSet Orchestration Mode VM (VMO). diff --git a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go index 0d573f77f..76374270f 100644 --- a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go @@ -729,6 +729,7 @@ func autoConvert_v1alpha1_MachineImage_To_azure_MachineImage(in *MachineImage, o out.SharedGalleryImageID = (*string)(unsafe.Pointer(in.SharedGalleryImageID)) out.AcceleratedNetworking = (*bool)(unsafe.Pointer(in.AcceleratedNetworking)) out.Architecture = (*string)(unsafe.Pointer(in.Architecture)) + out.SkipMarketplaceAgreement = (*bool)(unsafe.Pointer(in.SkipMarketplaceAgreement)) return nil } @@ -746,6 +747,7 @@ func autoConvert_azure_MachineImage_To_v1alpha1_MachineImage(in *azure.MachineIm out.SharedGalleryImageID = (*string)(unsafe.Pointer(in.SharedGalleryImageID)) out.AcceleratedNetworking = (*bool)(unsafe.Pointer(in.AcceleratedNetworking)) out.Architecture = (*string)(unsafe.Pointer(in.Architecture)) + out.SkipMarketplaceAgreement = (*bool)(unsafe.Pointer(in.SkipMarketplaceAgreement)) return nil } @@ -757,6 +759,7 @@ func Convert_azure_MachineImage_To_v1alpha1_MachineImage(in *azure.MachineImage, func autoConvert_v1alpha1_MachineImageVersion_To_azure_MachineImageVersion(in *MachineImageVersion, out *azure.MachineImageVersion, s conversion.Scope) error { out.Version = in.Version out.URN = (*string)(unsafe.Pointer(in.URN)) + out.SkipMarketplaceAgreement = (*bool)(unsafe.Pointer(in.SkipMarketplaceAgreement)) out.ID = (*string)(unsafe.Pointer(in.ID)) out.CommunityGalleryImageID = (*string)(unsafe.Pointer(in.CommunityGalleryImageID)) out.SharedGalleryImageID = (*string)(unsafe.Pointer(in.SharedGalleryImageID)) @@ -773,6 +776,7 @@ func Convert_v1alpha1_MachineImageVersion_To_azure_MachineImageVersion(in *Machi func autoConvert_azure_MachineImageVersion_To_v1alpha1_MachineImageVersion(in *azure.MachineImageVersion, out *MachineImageVersion, s conversion.Scope) error { out.Version = in.Version out.URN = (*string)(unsafe.Pointer(in.URN)) + out.SkipMarketplaceAgreement = (*bool)(unsafe.Pointer(in.SkipMarketplaceAgreement)) out.ID = (*string)(unsafe.Pointer(in.ID)) out.CommunityGalleryImageID = (*string)(unsafe.Pointer(in.CommunityGalleryImageID)) out.SharedGalleryImageID = (*string)(unsafe.Pointer(in.SharedGalleryImageID)) diff --git a/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go index 29e43aa48..f72c61c19 100644 --- a/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go @@ -452,6 +452,11 @@ func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = new(string) **out = **in } + if in.SkipMarketplaceAgreement != nil { + in, out := &in.SkipMarketplaceAgreement, &out.SkipMarketplaceAgreement + *out = new(bool) + **out = **in + } return } @@ -473,6 +478,11 @@ func (in *MachineImageVersion) DeepCopyInto(out *MachineImageVersion) { *out = new(string) **out = **in } + if in.SkipMarketplaceAgreement != nil { + in, out := &in.SkipMarketplaceAgreement, &out.SkipMarketplaceAgreement + *out = new(bool) + **out = **in + } if in.ID != nil { in, out := &in.ID, &out.ID *out = new(string) diff --git a/pkg/apis/azure/zz_generated.deepcopy.go b/pkg/apis/azure/zz_generated.deepcopy.go index 54007eb36..86510693c 100644 --- a/pkg/apis/azure/zz_generated.deepcopy.go +++ b/pkg/apis/azure/zz_generated.deepcopy.go @@ -452,6 +452,11 @@ func (in *MachineImage) DeepCopyInto(out *MachineImage) { *out = new(string) **out = **in } + if in.SkipMarketplaceAgreement != nil { + in, out := &in.SkipMarketplaceAgreement, &out.SkipMarketplaceAgreement + *out = new(bool) + **out = **in + } return } @@ -473,6 +478,11 @@ func (in *MachineImageVersion) DeepCopyInto(out *MachineImageVersion) { *out = new(string) **out = **in } + if in.SkipMarketplaceAgreement != nil { + in, out := &in.SkipMarketplaceAgreement, &out.SkipMarketplaceAgreement + *out = new(bool) + **out = **in + } if in.ID != nil { in, out := &in.ID, &out.ID *out = new(string) diff --git a/pkg/azure/types.go b/pkg/azure/types.go index 17f12c3b1..b4dd8dd3b 100644 --- a/pkg/azure/types.go +++ b/pkg/azure/types.go @@ -141,9 +141,6 @@ const ( SeedAnnotationKeyUseFlow = AnnotationKeyUseFlow // SeedAnnotationUseFlowValueNew is the value to restrict flow reconciliation to new shoot clusters SeedAnnotationUseFlowValueNew = "new" - - // BetaSkipMarketPlaceAgreementAnnotation when present will instruct the worker controller to create machine classes that skip the marketplace agreement by MCM. - BetaSkipMarketPlaceAgreementAnnotation = "beta.azure.provider.extensions.gardener.cloud/skip-marketplace-agreement" ) // UsernamePrefix is a constant for the username prefix of components deployed by Azure. diff --git a/pkg/controller/worker/machine_images.go b/pkg/controller/worker/machine_images.go index a265070cd..cf123f3c1 100644 --- a/pkg/controller/worker/machine_images.go +++ b/pkg/controller/worker/machine_images.go @@ -37,28 +37,28 @@ func (w *workerDelegate) UpdateMachineImagesStatus(ctx context.Context) error { return nil } -func (w *workerDelegate) findMachineImage(name, version string, architecture *string) (urn, id, communityGalleryImageID *string, sharedGalleryImageID *string, acceleratedNetworking *bool, err error) { +func (w *workerDelegate) findMachineImage(name, version string, architecture *string) (*api.MachineImage, error) { machineImage, err := helper.FindImageFromCloudProfile(w.cloudProfileConfig, name, version, architecture) if err == nil { - return machineImage.URN, machineImage.ID, machineImage.CommunityGalleryImageID, machineImage.SharedGalleryImageID, machineImage.AcceleratedNetworking, nil + return machineImage, nil } // Try to look up machine image in worker provider status as it was not found in componentconfig. if providerStatus := w.worker.Status.ProviderStatus; providerStatus != nil { workerStatus := &api.WorkerStatus{} if _, _, err := w.decoder.Decode(providerStatus.Raw, nil, workerStatus); err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("could not decode worker status of worker '%s': %w", kutil.ObjectName(w.worker), err) + return nil, fmt.Errorf("could not decode worker status of worker '%s': %w", kutil.ObjectName(w.worker), err) } machineImage, err := helper.FindMachineImage(workerStatus.MachineImages, name, version, architecture) if err != nil { - return nil, nil, nil, nil, nil, worker.ErrorMachineImageNotFound(name, version, *architecture) + return nil, worker.ErrorMachineImageNotFound(name, version, *architecture) } - return machineImage.URN, machineImage.ID, machineImage.CommunityGalleryImageID, machineImage.SharedGalleryImageID, machineImage.AcceleratedNetworking, nil + return machineImage, nil } - return nil, nil, nil, nil, nil, worker.ErrorMachineImageNotFound(name, version, *architecture) + return nil, worker.ErrorMachineImageNotFound(name, version, *architecture) } func appendMachineImage(machineImages []api.MachineImage, machineImage api.MachineImage) []api.MachineImage { diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 726d092e5..4039fad98 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -21,7 +21,6 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/utils" machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -88,7 +87,6 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { machineDeployments = worker.MachineDeployments{} machineClasses []map[string]interface{} machineImages []azureapi.MachineImage - skipAgreementPools = sets.New[string]() ) infrastructureStatus, err := w.decodeAzureInfrastructureStatus() @@ -106,12 +104,6 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { return err } - if v, ok := w.cluster.Shoot.GetAnnotations()[azure.BetaSkipMarketPlaceAgreementAnnotation]; ok { - for _, p := range strings.Split(v, ",") { - skipAgreementPools.Insert(p) - } - } - for _, pool := range w.worker.Spec.Pools { // Get the vmo dependency from the worker status if exists. vmoDependency, err := w.determineWorkerPoolVmoDependency(ctx, infrastructureStatus, workerStatus, pool.Name) @@ -121,33 +113,34 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { arch := ptr.Deref(pool.Architecture, v1beta1constants.ArchitectureAMD64) - urn, id, communityGalleryImageID, sharedGalleryImageID, imageSupportAcceleratedNetworking, err := w.findMachineImage(pool.MachineImage.Name, pool.MachineImage.Version, &arch) + machineImage, err := w.findMachineImage(pool.MachineImage.Name, pool.MachineImage.Version, &arch) if err != nil { return err } machineImages = appendMachineImage(machineImages, azureapi.MachineImage{ - Name: pool.MachineImage.Name, - Version: pool.MachineImage.Version, - URN: urn, - ID: id, - CommunityGalleryImageID: communityGalleryImageID, - SharedGalleryImageID: sharedGalleryImageID, - AcceleratedNetworking: imageSupportAcceleratedNetworking, - Architecture: &arch, + Name: pool.MachineImage.Name, + Version: pool.MachineImage.Version, + URN: machineImage.URN, + ID: machineImage.ID, + CommunityGalleryImageID: machineImage.CommunityGalleryImageID, + SharedGalleryImageID: machineImage.SharedGalleryImageID, + AcceleratedNetworking: machineImage.AcceleratedNetworking, + Architecture: &arch, + SkipMarketplaceAgreement: machineImage.SkipMarketplaceAgreement, }) image := map[string]interface{}{} - if urn != nil { - image["urn"] = *urn - if skipAgreementPools.Has(pool.Name) { - image["skipMarketplaceAgreement"] = true + if machineImage.URN != nil { + image["urn"] = *machineImage.URN + if ok := ptr.Deref(machineImage.SkipMarketplaceAgreement, false); ok { + image["skipMarketplaceAgreement"] = ok } - } else if communityGalleryImageID != nil { - image["communityGalleryImageID"] = *communityGalleryImageID - } else if sharedGalleryImageID != nil { - image["sharedGalleryImageID"] = *sharedGalleryImageID + } else if machineImage.CommunityGalleryImageID != nil { + image["communityGalleryImageID"] = *machineImage.CommunityGalleryImageID + } else if machineImage.SharedGalleryImageID != nil { + image["sharedGalleryImageID"] = *machineImage.SharedGalleryImageID } else { - image["id"] = *id + image["id"] = *machineImage.ID } disks, err := computeDisks(pool) @@ -204,7 +197,7 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error { if infrastructureStatus.Networks.VNet.ResourceGroup != nil { networkConfig["vnetResourceGroup"] = *infrastructureStatus.Networks.VNet.ResourceGroup } - if imageSupportAcceleratedNetworking != nil && *imageSupportAcceleratedNetworking && w.isMachineTypeSupportingAcceleratedNetworking(pool.MachineType) && acceleratedNetworkAllowed { + if ptr.Deref(machineImage.AcceleratedNetworking, false) && w.isMachineTypeSupportingAcceleratedNetworking(pool.MachineType) && acceleratedNetworkAllowed { networkConfig["acceleratedNetworking"] = true } machineClassSpec["network"] = networkConfig