[DO NOT MERGE] Encryption At Host Dynamic Validation

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
 	github.com/Azure/go-autorest/autorest v0.11.29

go.sum

@@ -22,6 +22,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 h1:wIDqH4WA5uJ6irRqjzodeSw6Pmp0tu3oIbwzBZEdMfQ=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0/go.mod h1:g8mnARUMaYRsg80mxm3PxjF7+oUotB/lneDbwYbGNxg=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=

pkg/frontend/encryptionathost_validation.go

@@ -0,0 +1,67 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+
+	sdk_armfeatures "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armfeatures"
+)
+
+type EncryptionAtHostValidator interface {
+	ValidateEncryptionAtHost(ctx context.Context, environment env.Interface, subscriptionID, tenantID string, oc *api.OpenShiftCluster) error
+}
+
+type encryptionAtHostValidator struct{}
+
+func (e encryptionAtHostValidator) ValidateEncryptionAtHost(ctx context.Context, environment env.Interface, subscriptionID, tenantID string, oc *api.OpenShiftCluster) error {
+	credential, err := environment.FPNewClientCertificateCredential(tenantID)
+	if err != nil {
+		return err
+	}
+
+	subFeatureRegistrationsClient, err := sdk_armfeatures.NewSubscriptionFeatureRegistrationsClient(subscriptionID, credential, nil)
+	if err != nil {
+		return err
+	}
+	return validateEncryptionAtHostGivenClient(ctx, subFeatureRegistrationsClient, oc)
+}
+
+func validateEncryptionAtHostGivenClient(ctx context.Context, subFeatureRegistrationsClient armfeatures.SubscriptionFeatureRegistrationsClient, oc *api.OpenShiftCluster) error {
+	var clusterUsesEncryptionAtHost = false
+	profilesToCheck := append([]api.WorkerProfile{{EncryptionAtHost: oc.Properties.MasterProfile.EncryptionAtHost}}, oc.Properties.WorkerProfiles...)
+	for _, profile := range profilesToCheck {
+		if profile.EncryptionAtHost == api.EncryptionAtHostEnabled {
+			clusterUsesEncryptionAtHost = true
+			break
+		}
+	}
+	if !clusterUsesEncryptionAtHost {
+		return nil
+	}
+	return validateSubscriptionIsRegisteredForEncryptionAtHost(ctx, subFeatureRegistrationsClient)
+}
+
+func validateSubscriptionIsRegisteredForEncryptionAtHost(ctx context.Context, subFeatureRegistrationsClient armfeatures.SubscriptionFeatureRegistrationsClient) error {
+	response, err := subFeatureRegistrationsClient.Get(ctx, "Microsoft.Compute", "EncryptionAtHost", nil)
+	if err != nil {
+		return err
+	}
+	if *response.Properties.State != sdk_armfeatures.SubscriptionFeatureRegistrationStateRegistered {
+		return &api.CloudError{
+			StatusCode: http.StatusBadRequest,
+			CloudErrorBody: &api.CloudErrorBody{
+				Code:    api.CloudErrorCodeInvalidParameter,
+				Message: "Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription. Register the feature using 'az feature register --namespace Microsoft.Compute --name EncryptionAtHost'",
+				Target:  "armfeatures.SubscriptionFeatureRegistrationProperties",
+			},
+		}
+	}
+	return nil
+}

pkg/frontend/encryptionathost_validation_test.go

@@ -0,0 +1,97 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures"
+	"github.com/golang/mock/gomock"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	mock_armfeatures "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/azuresdk/armfeatures"
+	utilerror "github.com/Azure/ARO-RP/test/util/error"
+)
+
+func TestValidateEncryptionAtHost(t *testing.T) {
+	getClusterWithNodeProfiles := func(MasterProfile api.MasterProfile, WorkerProfiles []api.WorkerProfile) *api.OpenShiftCluster {
+		return &api.OpenShiftCluster{
+			Properties: api.OpenShiftClusterProperties{
+				MasterProfile:  MasterProfile,
+				WorkerProfiles: WorkerProfiles,
+			},
+		}
+	}
+
+	getSubscriptionWithFeatureState := func(state armfeatures.SubscriptionFeatureRegistrationState) *armfeatures.SubscriptionFeatureRegistrationsClientGetResponse {
+		return &armfeatures.SubscriptionFeatureRegistrationsClientGetResponse{
+			SubscriptionFeatureRegistration: armfeatures.SubscriptionFeatureRegistration{
+				Properties: &armfeatures.SubscriptionFeatureRegistrationProperties{
+					State: &state,
+				},
+			},
+		}
+	}
+
+	for _, tt := range []struct {
+		name            string
+		oc              *api.OpenShiftCluster
+		mockResponse    *armfeatures.SubscriptionFeatureRegistrationsClientGetResponse
+		mockErr         error
+		wantErr         string
+		wantNumArmCalls int
+	}{
+		{
+			name:            "valid: cluster encryption at host disabled and subscription feature isn't registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostDisabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostDisabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateNotRegistered),
+			wantNumArmCalls: 0,
+		},
+		{
+			name:            "valid: cluster encryption at host disabled and subscription feature is registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostDisabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostDisabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateRegistered),
+			wantNumArmCalls: 0,
+		},
+		{
+			name:            "valid: cluster encryption at host enabled and subscription feature is registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostEnabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostEnabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateRegistered),
+			wantNumArmCalls: 1,
+		},
+		{
+			name:            "invalid: cluster master and worker encryption at host enabled and subscription feature isn't registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostEnabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostEnabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateNotRegistered),
+			wantErr:         "400: InvalidParameter: armfeatures.SubscriptionFeatureRegistrationProperties: Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription. Register the feature using 'az feature register --namespace Microsoft.Compute --name EncryptionAtHost'",
+			wantNumArmCalls: 1,
+		},
+		{
+			name:            "invalid: cluster master encryption at host enabled and subscription feature isn't registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostEnabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostDisabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateNotRegistered),
+			wantErr:         "400: InvalidParameter: armfeatures.SubscriptionFeatureRegistrationProperties: Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription. Register the feature using 'az feature register --namespace Microsoft.Compute --name EncryptionAtHost'",
+			wantNumArmCalls: 1,
+		},
+		{
+			name:            "invalid: cluster worker encryption at host enabled and subscription feature isn't registered",
+			oc:              getClusterWithNodeProfiles(api.MasterProfile{EncryptionAtHost: api.EncryptionAtHostDisabled}, []api.WorkerProfile{{EncryptionAtHost: api.EncryptionAtHostEnabled}}),
+			mockResponse:    getSubscriptionWithFeatureState(armfeatures.SubscriptionFeatureRegistrationStateNotRegistered),
+			wantErr:         "400: InvalidParameter: armfeatures.SubscriptionFeatureRegistrationProperties: Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription. Register the feature using 'az feature register --namespace Microsoft.Compute --name EncryptionAtHost'",
+			wantNumArmCalls: 1,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			subFeatureRegistrationsClient := mock_armfeatures.NewMockSubscriptionFeatureRegistrationsClient(controller)
+			subFeatureRegistrationsClient.EXPECT().Get(gomock.Any(), "Microsoft.Compute", "EncryptionAtHost", gomock.Any()).Return(*tt.mockResponse, tt.mockErr).Times(tt.wantNumArmCalls)
+
+			err := validateEncryptionAtHostGivenClient(context.Background(), subFeatureRegistrationsClient, tt.oc)
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+		})
+	}
+}

pkg/frontend/frontend.go

@@ -86,9 +86,10 @@ type frontend struct {
 	azureActionsFactory   azureActionsFactory
 	appLensActionsFactory appLensActionsFactory
 
-	skuValidator       SkuValidator
-	quotaValidator     QuotaValidator
-	providersValidator ProvidersValidator
+	skuValidator              SkuValidator
+	quotaValidator            QuotaValidator
+	providersValidator        ProvidersValidator
+	encryptionathostValidator EncryptionAtHostValidator
 
 	clusterEnricher clusterdata.BestEffortEnricher
 
@@ -161,9 +162,10 @@ func NewFrontend(ctx context.Context,
 		azureActionsFactory:   azureActionsFactory,
 		appLensActionsFactory: appLensActionsFactory,
 
-		quotaValidator:     quotaValidator{},
-		skuValidator:       skuValidator{},
-		providersValidator: providersValidator{},
+		quotaValidator:            quotaValidator{},
+		skuValidator:              skuValidator{},
+		providersValidator:        providersValidator{},
+		encryptionathostValidator: encryptionAtHostValidator{},
 
 		clusterEnricher: enricher,
 

pkg/frontend/openshiftcluster_putorpatch.go

@@ -387,6 +387,11 @@ func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.Sub
 		return err
 	}
 
+	err = f.encryptionathostValidator.ValidateEncryptionAtHost(ctx, f.env, subscription.ID, subscription.Subscription.Properties.TenantID, cluster)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

pkg/util/azureclient/azuresdk/armfeatures/generate.go

@@ -0,0 +1,8 @@
+package armfeatures
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate rm -rf ../../../../../pkg/util/mocks/azureclient/azuresdk/$GOPACKAGE
+//go:generate mockgen -destination=../../../mocks/azureclient/azuresdk/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/$GOPACKAGE SubscriptionFeatureRegistrationsClient
+//go:generate goimports -local=github.com/Azure/ARO-RP -e -w ../../../mocks/azureclient/azuresdk/$GOPACKAGE/$GOPACKAGE.go

pkg/util/azureclient/azuresdk/armfeatures/subscription_feature_registrations.go

@@ -0,0 +1,15 @@
+package armfeatures
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures"
+)
+
+// SubscriptionFeatureRegistrationsClient is a minimal interface for azure SubscriptionFeatureRegistrationsClient
+type SubscriptionFeatureRegistrationsClient interface {
+	Get(ctx context.Context, providerNamespace string, featureName string, options *armfeatures.SubscriptionFeatureRegistrationsClientGetOptions) (armfeatures.SubscriptionFeatureRegistrationsClientGetResponse, error)
+}