diff --git a/.vscode/launch.json b/.vscode/launch.json index 76ff9e18248..4c54285757e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,15 +1,15 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Attach to Process", - "type": "go", - "request": "attach", - "mode": "local", - "processId": "${command:pickGoProcess}" - } - ] -} + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Process", + "type": "go", + "request": "attach", + "mode": "local", + "processId": "${command:pickGoProcess}" + } + ] +} \ No newline at end of file diff --git a/cli/azd/pkg/azure/resource_ids.go b/cli/azd/pkg/azure/resource_ids.go index c5b056adf66..a02b221373e 100644 --- a/cli/azd/pkg/azure/resource_ids.go +++ b/cli/azd/pkg/azure/resource_ids.go @@ -51,6 +51,11 @@ func WebsiteRID(subscriptionId, resourceGroupName, websiteName string) string { return returnValue } +func AksRID(subscriptionId, resourceGroupName, clusterName string) string { + returnValue := fmt.Sprintf("%s/providers/Microsoft.ContainerService/managedClusters/%s", ResourceGroupRID(subscriptionId, resourceGroupName), clusterName) + return returnValue +} + func ContainerAppRID(subscriptionId, resourceGroupName, containerAppName string) string { returnValue := fmt.Sprintf( "%s/providers/Microsoft.App/containerApps/%s", diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index a07d31cf8fd..16a013d2fe7 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -36,6 +36,9 @@ const TenantIdEnvVarName = "AZURE_TENANT_ID" // to. const ContainerRegistryEndpointEnvVarName = "AZURE_CONTAINER_REGISTRY_ENDPOINT" +// ContainerRegistryEndpointEnvVarName is the name of they key used to store the endpoint of the container registry to push to. +const AksClusterEnvVarName = "AZURE_AKS_CLUSTER_NAME" + // ResourceGroupEnvVarName is the name of the azure resource group that should be used for deployments const ResourceGroupEnvVarName = "AZURE_RESOURCE_GROUP" diff --git a/cli/azd/pkg/exec/command_runner.go b/cli/azd/pkg/exec/command_runner.go index bd4961e8d12..585b331ba65 100644 --- a/cli/azd/pkg/exec/command_runner.go +++ b/cli/azd/pkg/exec/command_runner.go @@ -58,7 +58,14 @@ func (r *commandRunner) Run(ctx context.Context, args RunArgs) (RunResult, error cmd.Dir = args.Cwd - var stdin, stdout, stderr bytes.Buffer + var stdin io.Reader + if args.StdIn != nil { + stdin = args.StdIn + } else { + stdin = new(bytes.Buffer) + } + + var stdout, stderr bytes.Buffer cmd.Env = appendEnv(args.Env) @@ -67,7 +74,7 @@ func (r *commandRunner) Run(ctx context.Context, args RunArgs) (RunResult, error cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr } else { - cmd.Stdin = &stdin + cmd.Stdin = stdin cmd.Stdout = &stdout cmd.Stderr = &stderr diff --git a/cli/azd/pkg/exec/runargs.go b/cli/azd/pkg/exec/runargs.go index 7c121c5fbe3..ee370a14bdf 100644 --- a/cli/azd/pkg/exec/runargs.go +++ b/cli/azd/pkg/exec/runargs.go @@ -29,6 +29,9 @@ type RunArgs struct { // When set will attach commands to std input/output Interactive bool + + // When set will call the command with the specified StdIn + StdIn io.Reader } // NewRunArgs creates a new instance with the specified cmd and args @@ -81,3 +84,8 @@ func (b RunArgs) WithDebug(debug bool) RunArgs { b.Debug = debug return b } + +func (b RunArgs) WithStdIn(stdIn io.Reader) RunArgs { + b.StdIn = stdIn + return b +} diff --git a/cli/azd/pkg/infra/azure_resource_types.go b/cli/azd/pkg/infra/azure_resource_types.go index 92cccfc92f4..48ac7d52724 100644 --- a/cli/azd/pkg/infra/azure_resource_types.go +++ b/cli/azd/pkg/infra/azure_resource_types.go @@ -27,6 +27,8 @@ const ( AzureResourceTypeCacheForRedis AzureResourceType = "Microsoft.Cache/redis" AzureResourceTypePostgreSqlServer AzureResourceType = "Microsoft.DBforPostgreSQL/flexibleServers" AzureResourceTypeCDNProfile AzureResourceType = "Microsoft.Cdn/profiles" + AzureResourceTypeContainerRegistry AzureResourceType = "Microsoft.ContainerRegistry/registries" + AzureResourceTypeManagedCluster AzureResourceType = "Microsoft.ContainerService/managedClusters" ) const resourceLevelSeparator = "/" @@ -72,6 +74,10 @@ func GetResourceTypeDisplayName(resourceType AzureResourceType) string { return "Azure Database for PostgreSQL flexible server" case AzureResourceTypeCDNProfile: return "Azure Front Door / CDN profile" + case AzureResourceTypeContainerRegistry: + return "Container Registry" + case AzureResourceTypeManagedCluster: + return "AKS Managed Cluster" } return "" diff --git a/cli/azd/pkg/project/framework_service_docker.go b/cli/azd/pkg/project/framework_service_docker.go index 5b3a51a51c7..73ff59d4c3c 100644 --- a/cli/azd/pkg/project/framework_service_docker.go +++ b/cli/azd/pkg/project/framework_service_docker.go @@ -23,7 +23,7 @@ type DockerProjectOptions struct { type dockerProject struct { config *ServiceConfig env *environment.Environment - docker *docker.Docker + docker docker.Docker framework FrameworkService } @@ -66,7 +66,7 @@ func (p *dockerProject) Initialize(ctx context.Context) error { func NewDockerProject( config *ServiceConfig, env *environment.Environment, - docker *docker.Docker, + docker docker.Docker, framework FrameworkService, ) FrameworkService { return &dockerProject{ diff --git a/cli/azd/pkg/project/service_config.go b/cli/azd/pkg/project/service_config.go index 5a5e3526117..0051567639d 100644 --- a/cli/azd/pkg/project/service_config.go +++ b/cli/azd/pkg/project/service_config.go @@ -17,6 +17,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/tools" "github.com/azure/azure-dev/cli/azd/pkg/tools/azcli" "github.com/azure/azure-dev/cli/azd/pkg/tools/docker" + "github.com/azure/azure-dev/cli/azd/pkg/tools/kubectl" "github.com/azure/azure-dev/cli/azd/pkg/tools/swa" ) @@ -124,6 +125,12 @@ func (sc *ServiceConfig) GetServiceTarget( target, err = NewFunctionAppTarget(sc, env, resource, azCli) case string(StaticWebAppTarget): target, err = NewStaticWebAppTarget(sc, env, resource, azCli, swa.NewSwaCli(commandRunner)) + case string(AksTarget): + containerService, err := azCli.ContainerService(ctx, env.GetSubscriptionId()) + if err != nil { + return nil, err + } + target = NewAksTarget(sc, env, resource, azCli, containerService, kubectl.NewKubectl(commandRunner), docker.NewDocker(commandRunner)) default: return nil, fmt.Errorf("unsupported host '%s' for service '%s'", sc.Host, sc.Name) } @@ -154,7 +161,7 @@ func (sc *ServiceConfig) GetFrameworkService( } // For containerized applications we use a nested framework service - if sc.Host == string(ContainerAppTarget) { + if sc.Host == string(ContainerAppTarget) || sc.Host == string(AksTarget) { sourceFramework := frameworkService frameworkService = NewDockerProject(sc, env, docker.NewDocker(commandRunner), sourceFramework) } diff --git a/cli/azd/pkg/project/service_target.go b/cli/azd/pkg/project/service_target.go index 71cecbd61f1..d38c4761304 100644 --- a/cli/azd/pkg/project/service_target.go +++ b/cli/azd/pkg/project/service_target.go @@ -20,6 +20,7 @@ const ( ContainerAppTarget ServiceTargetKind = "containerapp" AzureFunctionTarget ServiceTargetKind = "function" StaticWebAppTarget ServiceTargetKind = "staticwebapp" + AksTarget ServiceTargetKind = "aks" ) type ServiceDeploymentResult struct { @@ -89,10 +90,5 @@ func resourceTypeMismatchError( // As an example, ContainerAppTarget is able to provision the container app as part of deployment, // and thus returns true. func (st ServiceTargetKind) SupportsDelayedProvisioning() bool { - return st == ContainerAppTarget + return st == ContainerAppTarget || st == AksTarget } - -var _ ServiceTarget = &appServiceTarget{} -var _ ServiceTarget = &containerAppTarget{} -var _ ServiceTarget = &functionAppTarget{} -var _ ServiceTarget = &staticWebAppTarget{} diff --git a/cli/azd/pkg/project/service_target_aks.go b/cli/azd/pkg/project/service_target_aks.go new file mode 100644 index 00000000000..a906b7e1907 --- /dev/null +++ b/cli/azd/pkg/project/service_target_aks.go @@ -0,0 +1,177 @@ +package project + +import ( + "context" + "fmt" + "log" + "path/filepath" + "time" + + "github.com/azure/azure-dev/cli/azd/pkg/azure" + "github.com/azure/azure-dev/cli/azd/pkg/environment" + "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" + "github.com/azure/azure-dev/cli/azd/pkg/tools" + "github.com/azure/azure-dev/cli/azd/pkg/tools/azcli" + "github.com/azure/azure-dev/cli/azd/pkg/tools/docker" + "github.com/azure/azure-dev/cli/azd/pkg/tools/kubectl" +) + +type aksTarget struct { + config *ServiceConfig + env *environment.Environment + scope *environment.TargetResource + containerService azcli.ContainerServiceClient + az azcli.AzCli + docker docker.Docker + kubectl kubectl.KubectlCli +} + +func (t *aksTarget) RequiredExternalTools() []tools.ExternalTool { + return []tools.ExternalTool{t.docker} +} + +func (t *aksTarget) Deploy(ctx context.Context, azdCtx *azdcontext.AzdContext, path string, progress chan<- string) (ServiceDeploymentResult, error) { + // Login to AKS cluster + namespace := t.config.Project.Name + clusterName, has := t.env.Values[environment.AksClusterEnvVarName] + if !has { + return ServiceDeploymentResult{}, fmt.Errorf("could not determine AKS cluster, ensure %s is set as an output of your infrastructure", environment.AksClusterEnvVarName) + } + + log.Printf("getting AKS credentials %s\n", clusterName) + progress <- "Getting AKS credentials" + credentials, err := t.containerService.GetAdminCredentials(ctx, t.scope.ResourceGroupName(), clusterName) + if err != nil { + return ServiceDeploymentResult{}, err + } + + kubeConfigManager, err := kubectl.NewKubeConfigManager(t.kubectl) + if err != nil { + return ServiceDeploymentResult{}, err + } + + kubeConfig, err := kubectl.ParseKubeConfig(ctx, credentials.Kubeconfigs[0].Value) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed parsing kube config: %w", err) + } + + if err := kubeConfigManager.SaveKubeConfig(ctx, clusterName, kubeConfig); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed saving kube config: %w", err) + } + + if err := kubeConfigManager.MergeConfigs(ctx, "config", "config", clusterName); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed merging kube configs: %w", err) + } + + if _, err := t.kubectl.ConfigUseContext(ctx, clusterName, nil); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed using kube context '%s', %w", clusterName, err) + } + + kubeFlags := kubectl.KubeCliFlags{ + Namespace: namespace, + DryRun: "client", + Output: "yaml", + } + + progress <- "Creating k8s namespace" + namespaceResult, err := t.kubectl.CreateNamespace(ctx, namespace, &kubectl.KubeCliFlags{DryRun: "client", Output: "yaml"}) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed creating kube namespace: %w", err) + } + + _, err = t.kubectl.ApplyPipe(ctx, *namespaceResult, nil) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed applying kube namespace: %w", err) + } + + progress <- "Creating k8s secrets" + secrets := t.env.Environ() + secretResult, err := t.kubectl.CreateSecretGenericFromLiterals(ctx, "azd", secrets, &kubeFlags) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed setting kube secrets: %w", err) + } + + _, err = t.kubectl.ApplyPipe(ctx, *secretResult, nil) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed applying kube secrets: %w", err) + } + + // Login to container registry. + loginServer, has := t.env.Values[environment.ContainerRegistryEndpointEnvVarName] + if !has { + return ServiceDeploymentResult{}, fmt.Errorf("could not determine container registry endpoint, ensure %s is set as an output of your infrastructure", environment.ContainerRegistryEndpointEnvVarName) + } + + log.Printf("logging into registry %s\n", loginServer) + + progress <- "Logging into container registry" + if err := t.az.LoginAcr(ctx, t.docker, t.env.GetSubscriptionId(), loginServer); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("logging into registry '%s': %w", loginServer, err) + } + + resourceName := t.scope.ResourceName() + if resourceName == "" { + resourceName = t.config.Name + } + + tags := []string{ + fmt.Sprintf("%s/%s/%s:azdev-deploy-%d", loginServer, t.config.Project.Name, resourceName, time.Now().Unix()), + fmt.Sprintf("%s/%s/%s:latest", loginServer, t.config.Project.Name, resourceName), + } + + for _, tag := range tags { + // Tag image. + log.Printf("tagging image %s as %s", path, tag) + progress <- "Tagging image" + if err := t.docker.Tag(ctx, t.config.Path(), path, tag); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("tagging image: %w", err) + } + + // Push image. + progress <- "Pushing container image" + if err := t.docker.Push(ctx, t.config.Path(), tag); err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("pushing image: %w", err) + } + } + + progress <- "Applying k8s manifests" + _, err = t.kubectl.ApplyFiles(ctx, filepath.Join(t.config.RelativePath, "manifests"), &kubectl.KubeCliFlags{Namespace: namespace}) + if err != nil { + return ServiceDeploymentResult{}, fmt.Errorf("failed applying kube manifests: %w", err) + } + + endpoints, err := t.Endpoints(ctx) + if err != nil { + return ServiceDeploymentResult{}, err + } + + return ServiceDeploymentResult{ + TargetResourceId: azure.ContainerAppRID(t.env.GetSubscriptionId(), t.scope.ResourceGroupName(), t.scope.ResourceName()), + Kind: ContainerAppTarget, + Details: nil, + Endpoints: endpoints, + }, nil +} + +func (t *aksTarget) Endpoints(ctx context.Context) ([]string, error) { + // TODO Update + return []string{"https://aks.azure.com/sample"}, nil + // containerAppProperties, err := t.cli.GetContainerAppProperties(ctx, t.env.GetSubscriptionId(), t.scope.ResourceGroupName(), t.scope.ResourceName()) + // if err != nil { + // return nil, fmt.Errorf("fetching service properties: %w", err) + // } + + // return []string{fmt.Sprintf("https://%s/", containerAppProperties.Properties.Configuration.Ingress.Fqdn)}, nil +} + +func NewAksTarget(config *ServiceConfig, env *environment.Environment, scope *environment.TargetResource, azCli azcli.AzCli, containerService azcli.ContainerServiceClient, kubectlCli kubectl.KubectlCli, docker docker.Docker) ServiceTarget { + return &aksTarget{ + config: config, + env: env, + scope: scope, + az: azCli, + containerService: containerService, + docker: docker, + kubectl: kubectlCli, + } +} diff --git a/cli/azd/pkg/project/service_target_containerapp.go b/cli/azd/pkg/project/service_target_containerapp.go index a5c96c4a71e..d985dd8d75b 100644 --- a/cli/azd/pkg/project/service_target_containerapp.go +++ b/cli/azd/pkg/project/service_target_containerapp.go @@ -31,7 +31,7 @@ type containerAppTarget struct { env *environment.Environment resource *environment.TargetResource cli azcli.AzCli - docker *docker.Docker + docker docker.Docker console input.Console commandRunner exec.CommandRunner accountManager account.Manager @@ -70,7 +70,7 @@ func (at *containerAppTarget) Deploy( log.Printf("logging into registry %s", loginServer) progress <- "Logging into container registry" - if err := at.cli.LoginAcr(ctx, at.commandRunner, at.env.GetSubscriptionId(), loginServer); err != nil { + if err := at.cli.LoginAcr(ctx, at.docker, at.env.GetSubscriptionId(), loginServer); err != nil { return ServiceDeploymentResult{}, fmt.Errorf("logging into registry '%s': %w", loginServer, err) } @@ -234,7 +234,7 @@ func NewContainerAppTarget( env *environment.Environment, resource *environment.TargetResource, azCli azcli.AzCli, - docker *docker.Docker, + docker docker.Docker, console input.Console, commandRunner exec.CommandRunner, accountManager account.Manager, diff --git a/cli/azd/pkg/tools/azcli/azcli.go b/cli/azd/pkg/tools/azcli/azcli.go index add99aa95b7..8548de6c067 100644 --- a/cli/azd/pkg/tools/azcli/azcli.go +++ b/cli/azd/pkg/tools/azcli/azcli.go @@ -16,8 +16,8 @@ import ( azdinternal "github.com/azure/azure-dev/cli/azd/internal" "github.com/azure/azure-dev/cli/azd/pkg/azsdk" "github.com/azure/azure-dev/cli/azd/pkg/azure" - "github.com/azure/azure-dev/cli/azd/pkg/exec" "github.com/azure/azure-dev/cli/azd/pkg/httputil" + "github.com/azure/azure-dev/cli/azd/pkg/tools/docker" ) var ( @@ -37,7 +37,7 @@ type AzCli interface { // UserAgent gets the currently configured user agent UserAgent() string - LoginAcr(ctx context.Context, commandRunner exec.CommandRunner, subscriptionId string, loginServer string) error + LoginAcr(ctx context.Context, dockerCli docker.Docker, subscriptionId string, loginServer string) error GetContainerRegistries(ctx context.Context, subscriptionId string) ([]*armcontainerregistry.Registry, error) GetSubscriptionDeployment( ctx context.Context, @@ -165,6 +165,9 @@ type AzCli interface { environmentName string, ) (*AzCliStaticWebAppEnvironmentProperties, error) GetAccessToken(ctx context.Context) (*AzCliAccessToken, error) + + // AKS + ContainerService(ctx context.Context, subscriptionId string) (ContainerServiceClient, error) } type AzCliDeployment struct { @@ -344,3 +347,22 @@ func clientOptionsBuilder(httpClient httputil.HttpClient, userAgent string) *azs WithTransport(httpClient). WithPerCallPolicy(azsdk.NewUserAgentPolicy(userAgent)) } + +func (cli *azCli) GetAksCredentials(ctx context.Context, resourceGroupName string, clusterName string) error { + // _, err := cli.runAzCommand(ctx, + // "aks", "get-credentials", + // "--resource-group", resourceGroupName, + // "--name", clusterName, + // ) + + // if err != nil { + // return fmt.Errorf("getting AKS credentials: %w", err) + // } + + return nil +} + +func (cli *azCli) ContainerService(ctx context.Context, subscriptionId string) (ContainerServiceClient, error) { + options := cli.createDefaultClientOptionsBuilder(ctx).BuildArmClientOptions() + return NewContainerServiceClient(subscriptionId, cli.credential, options) +} diff --git a/cli/azd/pkg/tools/azcli/container_registry.go b/cli/azd/pkg/tools/azcli/container_registry.go index db6c02030fb..80d874219c3 100644 --- a/cli/azd/pkg/tools/azcli/container_registry.go +++ b/cli/azd/pkg/tools/azcli/container_registry.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" "github.com/azure/azure-dev/cli/azd/pkg/azure" - "github.com/azure/azure-dev/cli/azd/pkg/exec" "github.com/azure/azure-dev/cli/azd/pkg/tools/docker" "golang.org/x/exp/slices" ) @@ -37,7 +36,7 @@ func (cli *azCli) GetContainerRegistries( } func (cli *azCli) LoginAcr(ctx context.Context, - commandRunner exec.CommandRunner, subscriptionId string, loginServer string, + dockerCli docker.Docker, subscriptionId string, loginServer string, ) error { client, err := cli.createRegistriesClient(ctx, subscriptionId) if err != nil { @@ -62,7 +61,6 @@ func (cli *azCli) LoginAcr(ctx context.Context, username := *credResponse.Username // Login to docker with ACR credentials to allow push operations - dockerCli := docker.NewDocker(commandRunner) err = dockerCli.Login(ctx, loginServer, username, *credResponse.Passwords[0].Value) if err != nil { return fmt.Errorf("failed logging into docker for username '%s' and server %s: %w", loginServer, username, err) diff --git a/cli/azd/pkg/tools/azcli/container_service.go b/cli/azd/pkg/tools/azcli/container_service.go new file mode 100644 index 00000000000..acfdefbf82d --- /dev/null +++ b/cli/azd/pkg/tools/azcli/container_service.go @@ -0,0 +1,39 @@ +package azcli + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2" +) + +type ContainerServiceClient interface { + GetAdminCredentials(ctx context.Context, resourceGroupName string, resourceName string) (*armcontainerservice.CredentialResults, error) +} + +type containerServiceClient struct { + client *armcontainerservice.ManagedClustersClient + subscriptionId string +} + +func NewContainerServiceClient(subscriptionId string, credential azcore.TokenCredential, options *arm.ClientOptions) (ContainerServiceClient, error) { + azureClient, err := armcontainerservice.NewManagedClustersClient(subscriptionId, credential, options) + if err != nil { + return nil, err + } + + return &containerServiceClient{ + subscriptionId: subscriptionId, + client: azureClient, + }, nil +} + +func (cs *containerServiceClient) GetAdminCredentials(ctx context.Context, resourceGroupName string, resourceName string) (*armcontainerservice.CredentialResults, error) { + creds, err := cs.client.ListClusterAdminCredentials(ctx, resourceGroupName, resourceName, nil) + if err != nil { + return nil, err + } + + return &creds.CredentialResults, nil +} diff --git a/cli/azd/pkg/tools/docker/docker.go b/cli/azd/pkg/tools/docker/docker.go index 76b62f00e8d..eb0edc8687f 100644 --- a/cli/azd/pkg/tools/docker/docker.go +++ b/cli/azd/pkg/tools/docker/docker.go @@ -13,17 +13,25 @@ import ( "github.com/blang/semver/v4" ) -func NewDocker(commandRunner exec.CommandRunner) *Docker { - return &Docker{ +type Docker interface { + tools.ExternalTool + Login(ctx context.Context, loginServer string, username string, password string) error + Build(ctx context.Context, cwd string, dockerFilePath string, platform string, buildContext string) (string, error) + Tag(ctx context.Context, cwd string, imageName string, tag string) error + Push(ctx context.Context, cwd string, tag string) error +} + +func NewDocker(commandRunner exec.CommandRunner) Docker { + return &docker{ commandRunner: commandRunner, } } -type Docker struct { +type docker struct { commandRunner exec.CommandRunner } -func (d *Docker) Login(ctx context.Context, loginServer string, username string, password string) error { +func (d *docker) Login(ctx context.Context, loginServer string, username string, password string) error { _, err := d.executeCommand(ctx, ".", "login", "--username", username, "--password", password, @@ -40,7 +48,7 @@ func (d *Docker) Login(ctx context.Context, loginServer string, username string, // it defaults to amd64. If the build // is successful, the function // returns the image id of the built image. -func (d *Docker) Build( +func (d *docker) Build( ctx context.Context, cwd string, dockerFilePath string, @@ -59,7 +67,7 @@ func (d *Docker) Build( return strings.TrimSpace(res.Stdout), nil } -func (d *Docker) Tag(ctx context.Context, cwd string, imageName string, tag string) error { +func (d *docker) Tag(ctx context.Context, cwd string, imageName string, tag string) error { res, err := d.executeCommand(ctx, cwd, "tag", imageName, tag) if err != nil { return fmt.Errorf("tagging image: %s: %w", res.String(), err) @@ -68,7 +76,7 @@ func (d *Docker) Tag(ctx context.Context, cwd string, imageName string, tag stri return nil } -func (d *Docker) Push(ctx context.Context, cwd string, tag string) error { +func (d *docker) Push(ctx context.Context, cwd string, tag string) error { res, err := d.executeCommand(ctx, cwd, "push", tag) if err != nil { return fmt.Errorf("pushing image: %s: %w", res.String(), err) @@ -77,7 +85,7 @@ func (d *Docker) Push(ctx context.Context, cwd string, tag string) error { return nil } -func (d *Docker) versionInfo() tools.VersionInfo { +func (d *docker) versionInfo() tools.VersionInfo { return tools.VersionInfo{ MinimumVersion: semver.Version{ Major: 17, @@ -162,8 +170,7 @@ func isSupportedDockerVersion(cliOutput string) (bool, error) { // If we reach this point, we don't understand how to validate the version based on its scheme. return false, fmt.Errorf("could not determine version from docker version string: %s", version) } - -func (d *Docker) CheckInstalled(ctx context.Context) (bool, error) { +func (d *docker) CheckInstalled(ctx context.Context) (bool, error) { found, err := tools.ToolInPath("docker") if !found { return false, err @@ -182,15 +189,15 @@ func (d *Docker) CheckInstalled(ctx context.Context) (bool, error) { return true, nil } -func (d *Docker) InstallUrl() string { +func (d *docker) InstallUrl() string { return "https://aka.ms/azure-dev/docker-install" } -func (d *Docker) Name() string { +func (d *docker) Name() string { return "Docker" } -func (d *Docker) executeCommand(ctx context.Context, cwd string, args ...string) (exec.RunResult, error) { +func (d *docker) executeCommand(ctx context.Context, cwd string, args ...string) (exec.RunResult, error) { runArgs := exec.NewRunArgs("docker", args...). WithCwd(cwd). WithEnrichError(true) diff --git a/cli/azd/pkg/tools/kubectl/kube_config.go b/cli/azd/pkg/tools/kubectl/kube_config.go new file mode 100644 index 00000000000..f9af9ffc513 --- /dev/null +++ b/cli/azd/pkg/tools/kubectl/kube_config.go @@ -0,0 +1,145 @@ +package kubectl + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/osutil" + "gopkg.in/yaml.v3" +) + +type KubeConfig struct { + ApiVersion string `yaml:"apiVersion"` + Clusters []*KubeCluster `yaml:"clusters"` + Contexts []*KubeContext `yaml:"contexts"` + Users []*KubeUser `yaml:"users"` + Kind string `yaml:"kind"` + CurrentContext string `yaml:"current-context"` + Preferences KubePreferences `yaml:"preferences"` +} + +type KubeCluster struct { + Name string `yaml:"name"` + Cluster KubeClusterData `yaml:"cluster"` +} + +type KubeClusterData struct { + CertificateAuthorityData string `yaml:"certificate-authority-data"` + Server string `yaml:"server"` +} + +type KubeContext struct { + Name string `yaml:"name"` + Context KubeContextData `yaml:"context"` +} + +type KubeContextData struct { + Cluster string `yaml:"cluster"` + User string `yaml:"user"` +} + +type KubeUser struct { + Name string `yaml:"name"` + KubeUserData KubeUserData `yaml:"user"` +} + +type KubeUserData map[string]any +type KubePreferences map[string]any + +type KubeConfigManager struct { + cli KubectlCli + configPath string +} + +func NewKubeConfigManager(cli KubectlCli) (*KubeConfigManager, error) { + kubeConfigDir, err := getKubeConfigDir() + if err != nil { + return nil, err + } + + return &KubeConfigManager{ + cli: cli, + configPath: kubeConfigDir, + }, nil +} + +func ParseKubeConfig(ctx context.Context, raw []byte) (*KubeConfig, error) { + var existing KubeConfig + if err := yaml.Unmarshal(raw, &existing); err != nil { + return nil, fmt.Errorf("failed unmarshalling Kube Config YAML: %w", err) + } + + return &existing, nil +} + +func (kcm *KubeConfigManager) SaveKubeConfig(ctx context.Context, configName string, config *KubeConfig) error { + kubeConfigRaw, err := yaml.Marshal(config) + if err != nil { + return fmt.Errorf("failed marshalling KubeConfig to yaml: %w", err) + } + + outFilePath := filepath.Join(kcm.configPath, configName) + err = os.WriteFile(outFilePath, kubeConfigRaw, osutil.PermissionFile) + if err != nil { + return fmt.Errorf("failed write kube config file: %w", err) + } + + return nil +} + +func (kcm *KubeConfigManager) DeleteKubeConfig(ctx context.Context, configName string) error { + kubeConfigPath := filepath.Join(kcm.configPath, configName) + err := os.Remove(kubeConfigPath) + if err != nil { + return fmt.Errorf("failed deleting kube config file: %w", err) + } + + return nil +} + +func (kcm *KubeConfigManager) MergeConfigs(ctx context.Context, newConfigName string, path ...string) error { + fullConfigPaths := []string{} + for _, kubeConfigName := range path { + fullConfigPaths = append(fullConfigPaths, filepath.Join(kcm.configPath, kubeConfigName)) + } + + kcm.cli.SetEnv(fmt.Sprintf("KUBECONFIG=%s", strings.Join(fullConfigPaths, string(os.PathListSeparator)))) + res, err := kcm.cli.ConfigView(ctx, true, true, nil) + if err != nil { + return fmt.Errorf("kubectl config view failed: %w", err) + } + + kubeConfigRaw := []byte(res.Stdout) + outFilePath := filepath.Join(kcm.configPath, newConfigName) + err = os.WriteFile(outFilePath, kubeConfigRaw, osutil.PermissionFile) + if err != nil { + return fmt.Errorf("failed writing new kube config: %w", err) + } + + return nil +} + +func (kcm *KubeConfigManager) AddOrUpdateContext(ctx context.Context, contextName string, newKubeConfig *KubeConfig) error { + err := kcm.SaveKubeConfig(ctx, contextName, newKubeConfig) + if err != nil { + return fmt.Errorf("failed write new kube context file: %w", err) + } + + err = kcm.MergeConfigs(ctx, "config", contextName) + if err != nil { + return fmt.Errorf("failed merging KUBE configs: %w", err) + } + + return nil +} + +func getKubeConfigDir() (string, error) { + userHomeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("cannot get user home directory: %w", err) + } + return filepath.Join(userHomeDir, ".kube"), nil +} diff --git a/cli/azd/pkg/tools/kubectl/kubectl.go b/cli/azd/pkg/tools/kubectl/kubectl.go new file mode 100644 index 00000000000..60c73ceb84d --- /dev/null +++ b/cli/azd/pkg/tools/kubectl/kubectl.go @@ -0,0 +1,211 @@ +package kubectl + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/exec" + "github.com/azure/azure-dev/cli/azd/pkg/tools" +) + +type KubectlCli interface { + tools.ExternalTool + Cwd(cwd string) + SetEnv(env ...string) + GetNodes(ctx context.Context, flags *KubeCliFlags) ([]Node, error) + ApplyFiles(ctx context.Context, path string, flags *KubeCliFlags) (*exec.RunResult, error) + ApplyKustomize(ctx context.Context, path string, flags *KubeCliFlags) (*exec.RunResult, error) + ConfigView(ctx context.Context, merge bool, flatten bool, flags *KubeCliFlags) (*exec.RunResult, error) + ConfigUseContext(ctx context.Context, name string, flags *KubeCliFlags) (*exec.RunResult, error) + CreateNamespace(ctx context.Context, name string, flags *KubeCliFlags) (*exec.RunResult, error) + CreateSecretGenericFromLiterals(ctx context.Context, name string, secrets []string, flags *KubeCliFlags) (*exec.RunResult, error) + ApplyPipe(ctx context.Context, result exec.RunResult, flags *KubeCliFlags) (*exec.RunResult, error) +} + +type kubectlCli struct { + tools.ExternalTool + commandRunner exec.CommandRunner + env []string + cwd string +} + +func (cli *kubectlCli) CheckInstalled(ctx context.Context) (bool, error) { + return true, nil +} + +func (cli *kubectlCli) InstallUrl() string { + return "https://aka.ms/azure-dev/kubectl-install" +} + +func (cli *kubectlCli) Name() string { + return "kubectl" +} + +func (cli *kubectlCli) SetEnv(env ...string) { + cli.env = env +} + +func (cli *kubectlCli) Cwd(cwd string) { + cli.cwd = cwd +} + +func (cli *kubectlCli) ConfigUseContext(ctx context.Context, name string, flags *KubeCliFlags) (*exec.RunResult, error) { + res, err := cli.executeCommand(ctx, flags, "config", "use-context", name) + if err != nil { + return nil, fmt.Errorf("failed setting kubectl context: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) ConfigView(ctx context.Context, merge bool, flatten bool, flags *KubeCliFlags) (*exec.RunResult, error) { + kubeConfigDir, err := getKubeConfigDir() + if err != nil { + return nil, err + } + + args := []string{"config", "view"} + if merge { + args = append(args, "--merge") + } + if flatten { + args = append(args, "--flatten") + } + + runArgs := exec.NewRunArgs("kubectl", args...). + WithCwd(kubeConfigDir). + WithEnv(cli.env) + + res, err := cli.executeCommandWithArgs(ctx, runArgs, flags) + if err != nil { + return nil, fmt.Errorf("kubectl config view: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) GetNodes(ctx context.Context, flags *KubeCliFlags) ([]Node, error) { + res, err := cli.executeCommand(ctx, flags, + "get", "nodes", + "-o", "json", + ) + if err != nil { + return nil, fmt.Errorf("kubectl get nodes: %w", err) + } + + var listResult ListResult + if err := json.Unmarshal([]byte(res.Stdout), &listResult); err != nil { + return nil, fmt.Errorf("unmarshaling json: %w", err) + } + + nodes := []Node{} + for _, item := range listResult.Items { + metadata := item["metadata"].(map[string]any) + + nodes = append(nodes, Node{ + Name: metadata["name"].(string), + }) + } + + return nodes, nil +} + +func (cli *kubectlCli) ApplyPipe(ctx context.Context, result exec.RunResult, flags *KubeCliFlags) (*exec.RunResult, error) { + runArgs := exec. + NewRunArgs("kubectl", "apply", "-f", "-"). + WithStdIn(strings.NewReader(result.Stdout)) + + res, err := cli.executeCommandWithArgs(ctx, runArgs, flags) + if err != nil { + return nil, fmt.Errorf("kubectl apply -f: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) ApplyFiles(ctx context.Context, path string, flags *KubeCliFlags) (*exec.RunResult, error) { + + res, err := cli.executeCommand(ctx, flags, "apply", "-f", path) + if err != nil { + return nil, fmt.Errorf("kubectl apply -f: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) ApplyKustomize(ctx context.Context, path string, flags *KubeCliFlags) (*exec.RunResult, error) { + res, err := cli.executeCommand(ctx, flags, "apply", "-k", path) + if err != nil { + return nil, fmt.Errorf("kubectl apply -k: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) CreateSecretGenericFromLiterals(ctx context.Context, name string, secrets []string, flags *KubeCliFlags) (*exec.RunResult, error) { + args := []string{"create", "secret", "generic", name} + for _, secret := range secrets { + args = append(args, fmt.Sprintf("--from-literal=%s", secret)) + } + + res, err := cli.executeCommand(ctx, flags, args...) + if err != nil { + return nil, fmt.Errorf("kubectl create secret generic --from-env-file: %w", err) + } + + return &res, nil +} + +type KubeCliFlags struct { + Namespace string + DryRun string + Output string +} + +func (cli *kubectlCli) CreateNamespace(ctx context.Context, name string, flags *KubeCliFlags) (*exec.RunResult, error) { + args := []string{"create", "namespace", name} + + res, err := cli.executeCommand(ctx, flags, args...) + if err != nil { + return nil, fmt.Errorf("kubectl create namespace: %w", err) + } + + return &res, nil +} + +func (cli *kubectlCli) executeCommand(ctx context.Context, flags *KubeCliFlags, args ...string) (exec.RunResult, error) { + runArgs := exec. + NewRunArgs("kubectl"). + AppendParams(args...) + + return cli.executeCommandWithArgs(ctx, runArgs, flags) +} + +func (cli *kubectlCli) executeCommandWithArgs(ctx context.Context, args exec.RunArgs, flags *KubeCliFlags) (exec.RunResult, error) { + args = args.WithEnrichError(true) + if cli.cwd != "" { + args = args.WithCwd(cli.cwd) + } + + if flags != nil { + if flags.DryRun != "" { + args = args.AppendParams(fmt.Sprintf("--dry-run=%s", flags.DryRun)) + } + if flags.Namespace != "" { + args = args.AppendParams("-n", flags.Namespace) + } + if flags.Output != "" { + args = args.AppendParams("-o", flags.Output) + } + } + + return cli.commandRunner.Run(ctx, args) +} + +func NewKubectl(commandRunner exec.CommandRunner) KubectlCli { + return &kubectlCli{ + commandRunner: commandRunner, + } +} diff --git a/cli/azd/pkg/tools/kubectl/kubectl_test.go b/cli/azd/pkg/tools/kubectl/kubectl_test.go new file mode 100644 index 00000000000..9642a7e8e1e --- /dev/null +++ b/cli/azd/pkg/tools/kubectl/kubectl_test.go @@ -0,0 +1,74 @@ +package kubectl + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/exec" + "github.com/azure/azure-dev/cli/azd/test/mocks" + "github.com/stretchr/testify/require" +) + +func Test_MergeKubeConfig(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + commandRunner := exec.NewCommandRunner(os.Stdin, os.Stdout, os.Stderr) + cli := NewKubectl(commandRunner) + kubeConfigManager, err := NewKubeConfigManager(cli) + require.NoError(t, err) + + config1 := createTestCluster("cluster1", "user1") + config2 := createTestCluster("cluster2", "user2") + config3 := createTestCluster("cluster3", "user3") + + defer func() { + err := kubeConfigManager.DeleteKubeConfig(*mockContext.Context, "config1") + require.NoError(t, err) + err = kubeConfigManager.DeleteKubeConfig(*mockContext.Context, "config2") + require.NoError(t, err) + err = kubeConfigManager.DeleteKubeConfig(*mockContext.Context, "config3") + require.NoError(t, err) + }() + + err = kubeConfigManager.SaveKubeConfig(*mockContext.Context, "config1", config1) + require.NoError(t, err) + err = kubeConfigManager.SaveKubeConfig(*mockContext.Context, "config2", config2) + require.NoError(t, err) + err = kubeConfigManager.SaveKubeConfig(*mockContext.Context, "config3", config3) + require.NoError(t, err) + + err = kubeConfigManager.MergeConfigs(*mockContext.Context, "config", "config1", "config2", "config3") + require.NoError(t, err) +} + +func createTestCluster(clusterName, username string) *KubeConfig { + return &KubeConfig{ + ApiVersion: "v1", + Kind: "Config", + CurrentContext: clusterName, + Preferences: KubePreferences{}, + Clusters: []*KubeCluster{ + { + Name: clusterName, + Cluster: KubeClusterData{ + Server: fmt.Sprintf("https://%s.eastus2.azmk8s.io:443", clusterName), + }, + }, + }, + Users: []*KubeUser{ + { + Name: fmt.Sprintf("%s_%s", clusterName, username), + }, + }, + Contexts: []*KubeContext{ + { + Name: clusterName, + Context: KubeContextData{ + Cluster: clusterName, + User: fmt.Sprintf("%s_%s", clusterName, username), + }, + }, + }, + } +} diff --git a/cli/azd/pkg/tools/kubectl/models.go b/cli/azd/pkg/tools/kubectl/models.go new file mode 100644 index 00000000000..7074775f22b --- /dev/null +++ b/cli/azd/pkg/tools/kubectl/models.go @@ -0,0 +1,14 @@ +package kubectl + +type Node struct { + Name string + Status string + Roles []string + Version string +} + +type ListResult struct { + ApiVersion string `json:"apiVersion"` + Kind string `json: "kind"` + Items []map[string]any `json:"items"` +} diff --git a/go.mod b/go.mod index 90c3e98a82c..448baa000e5 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,10 @@ require ( ) +require ( + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2 v2.2.0 // indirect + github.com/kr/text v0.2.0 // indirect +) require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect diff --git a/go.sum b/go.sum index 3e3a3555a6d..0bc9efb22ad 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization v1.0.0/go.mod h1:lPneRe3TwsoDRKY4O6YDLXHhEWrD+TIRa8XrV/3/fqw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v0.6.0 h1:Z5/bDxQL2Zc9t6ZDwdRU60bpLHZvoKOeuaM7XVbf2z0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v0.6.0/go.mod h1:0FPu3oDRGPvuX1H8TtHJ5XGA0KrXLunomcixR+PQGGA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2 v2.2.0 h1:3L+gX5ssCABAToH0VQ64/oNz7rr+ShW+2sB+sonzIlY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2 v2.2.0/go.mod h1:4gUds0dEPFIld6DwHfbo0cLBljyIyI5E5ciPb5MLi3Q= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.0.0 h1:Jc2KcpCDMu7wJfkrzn7fs/53QMDXH78GuqnH4HOd7zs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.0.0/go.mod h1:PFVgFsclKzPqYRT/BiwpfUN22cab0C7FlgXR3iWpwMo= diff --git a/templates/common/infra/bicep/core/host/aks/acragentpool.bicep b/templates/common/infra/bicep/core/host/aks/acragentpool.bicep new file mode 100644 index 00000000000..4bba2c43484 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/acragentpool.bicep @@ -0,0 +1,19 @@ +param location string = resourceGroup().location +param acrName string +param acrPoolSubnetId string = '' + +resource acr 'Microsoft.ContainerRegistry/registries@2021-06-01-preview' existing = { + name: acrName +} + +resource acrPool 'Microsoft.ContainerRegistry/registries/agentPools@2019-06-01-preview' = { + name: 'private-pool' + location: location + parent: acr + properties: { + count: 1 + os: 'Linux' + tier: 'S1' + virtualNetworkSubnetResourceId: acrPoolSubnetId + } +} diff --git a/templates/common/infra/bicep/core/host/aks/aksagentpool.bicep b/templates/common/infra/bicep/core/host/aks/aksagentpool.bicep new file mode 100644 index 00000000000..100992aaf88 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/aksagentpool.bicep @@ -0,0 +1,80 @@ +param AksName string + +param PoolName string + +@description('The zones to use for a node pool') +param availabilityZones array = [] + +@description('OS disk type') +param osDiskType string + +@description('VM SKU') +param agentVMSize string + +@description('Disk size in GB') +param osDiskSizeGB int = 0 + +@description('The number of agents for the user node pool') +param agentCount int = 1 + +@description('The maximum number of nodes for the user node pool') +param agentCountMax int = 3 +var autoScale = agentCountMax > agentCount + +@description('The maximum number of pods per node.') +param maxPods int = 30 + +@description('Any taints that should be applied to the node pool') +param nodeTaints array = [] + +@description('Any labels that should be applied to the node pool') +param nodeLabels object = {} + +@description('The subnet the node pool will use') +param subnetId string + +@description('OS Type for the node pool') +@allowed(['Linux','Windows']) +param osType string + +@allowed(['Ubuntu','Windows2019','Windows2022']) +param osSKU string + +@description('Assign a public IP per node') +param enableNodePublicIP bool = false + +@description('Apply a default sku taint to Windows node pools') +param autoTaintWindows bool = false + +var taints = autoTaintWindows ? union(nodeTaints, ['sku=Windows:NoSchedule']) : nodeTaints + +resource aks 'Microsoft.ContainerService/managedClusters@2021-10-01' existing = { + name: AksName +} + +resource userNodepool 'Microsoft.ContainerService/managedClusters/agentPools@2021-10-01' = { + parent: aks + name: PoolName + properties: { + mode: 'User' + vmSize: agentVMSize + count: agentCount + minCount: autoScale ? agentCount : json('null') + maxCount: autoScale ? agentCountMax : json('null') + enableAutoScaling: autoScale + availabilityZones: !empty(availabilityZones) ? availabilityZones : null + osDiskType: osDiskType + osSKU: osSKU + osDiskSizeGB: osDiskSizeGB + osType: osType + maxPods: maxPods + type: 'VirtualMachineScaleSets' + vnetSubnetID: !empty(subnetId) ? subnetId : json('null') + upgradeSettings: { + maxSurge: '33%' + } + nodeTaints: taints + nodeLabels: nodeLabels + enableNodePublicIP: enableNodePublicIP + } +} diff --git a/templates/common/infra/bicep/core/host/aks/aksmetricalerts.bicep b/templates/common/infra/bicep/core/host/aks/aksmetricalerts.bicep new file mode 100644 index 00000000000..d1ed20d9a01 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/aksmetricalerts.bicep @@ -0,0 +1,753 @@ +@description('The name of the AKS Cluster to configure the alerts on') +param clusterName string + +@description('The name of the Log Analytics workspace to log metric data to') +param logAnalyticsWorkspaceName string + +@description('The location of the Log Analytics workspace') +param logAnalyticsWorkspaceLocation string = resourceGroup().location + +@description('Select the frequency on how often the alert rule should be run. Selecting frequency smaller than granularity of datapoints grouping will result in sliding window evaluation') +@allowed([ + 'PT1M' + 'PT15M' +]) +param evalFrequency string = 'PT1M' + +@description('Create the metric alerts as either enabled or disabled') +param metricAlertsEnabled bool = true + +@description('Defines the interval over which datapoints are grouped using the aggregation type function') +@allowed([ + 'PT5M' + 'PT1H' +]) +param windowSize string = 'PT5M' + +@allowed([ + 'Critical' + 'Error' + 'Warning' + 'Informational' + 'Verbose' +]) +param alertSeverity string = 'Informational' + +var alertServerityLookup = { + Critical: 0 + Error: 1 + Warning: 2 + Informational: 3 + Verbose: 4 +} +var alertSeverityNumber = alertServerityLookup[alertSeverity] + +var AksResourceId = resourceId('Microsoft.ContainerService/managedClusters', clusterName) + +resource Node_CPU_utilization_high_for_clusterName_CI_1 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node CPU utilization high for ${clusterName} CI-1' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuUsagePercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node CPU utilization across the cluster.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Node_working_set_memory_utilization_high_for_clusterName_CI_2 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Node working set memory utilization high for ${clusterName} CI-2' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node working set memory utilization across the cluster.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Jobs_completed_more_than_6_hours_ago_for_clusterName_CI_11 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Jobs completed more than 6 hours ago for ${clusterName} CI-11' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'completedJobsCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors completed jobs (more than 6 hours ago).' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Container_CPU_usage_high_for_clusterName_CI_9 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container CPU usage high for ${clusterName} CI-9' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'cpuExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 90 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container CPU utilization.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Container_working_set_memory_usage_high_for_clusterName_CI_10 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Container working set memory usage high for ${clusterName} CI-10' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'memoryWorkingSetExceededPercentage' + metricNamespace: 'Insights.Container/containers' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 90 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors container working set memory utilization.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Pods_in_failed_state_for_clusterName_CI_4 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods in failed state for ${clusterName} CI-4' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'phase' + operator: 'Include' + values: [ + 'Failed' + ] + } + ] + metricName: 'podCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Pod status monitoring.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Disk_usage_high_for_clusterName_CI_5 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Disk usage high for ${clusterName} CI-5' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'host' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'device' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'DiskUsedPercentage' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors disk usage for all nodes and storage devices.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Nodes_in_not_ready_status_for_clusterName_CI_3 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Nodes in not ready status for ${clusterName} CI-3' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'status' + operator: 'Include' + values: [ + 'NotReady' + ] + } + ] + metricName: 'nodesCount' + metricNamespace: 'Insights.Container/nodes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'Node status monitoring.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Containers_getting_OOM_killed_for_clusterName_CI_6 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Containers getting OOM killed for ${clusterName} CI-6' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'oomKilledContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers killed due to out of memory (OOM) error.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Persistent_volume_usage_high_for_clusterName_CI_18 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Persistent volume usage high for ${clusterName} CI-18' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'podName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetesNamespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'pvUsageExceededPercentage' + metricNamespace: 'Insights.Container/persistentvolumes' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors persistent volume utilization.' + enabled: false + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Pods_not_in_ready_state_for_clusterName_CI_8 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Pods not in ready state for ${clusterName} CI-8' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'PodReadyPercentage' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'LessThan' + threshold: 80 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors for excessive pods not in the ready state.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'microsoft.containerservice/managedclusters' + windowSize: windowSize + } +} + +resource Restarting_container_count_for_clusterName_CI_7 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Restarting container count for ${clusterName} CI-7' + location: 'global' + properties: { + criteria: { + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + dimensions: [ + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + ] + metricName: 'restartingContainerCount' + metricNamespace: 'Insights.Container/pods' + name: 'Metric1' + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Average' + skipMetricValidation: true + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + description: 'This alert monitors number of containers restarting across the cluster.' + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + AksResourceId + ] + severity: alertSeverityNumber + targetResourceType: 'Microsoft.ContainerService/managedClusters' + windowSize: windowSize + } +} + +resource Container_CPU_usage_violates_the_configured_threshold_for_clustername_CI_19 'microsoft.insights/metricAlerts@2018-03-01' = { + name: 'Container CPU usage violates the configured threshold for ${clusterName} CI-19' + location: 'global' + properties: { + description: 'This alert monitors container CPU usage. It uses the threshold defined in the config map.' + severity: alertSeverityNumber + enabled: true + scopes: [ + AksResourceId + ] + evaluationFrequency: evalFrequency + windowSize: windowSize + criteria: { + allOf: [ + { + threshold: 0 + name: 'Metric1' + metricNamespace: 'Insights.Container/containers' + metricName: 'cpuThresholdViolated' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + operator: 'GreaterThan' + timeAggregation: 'Average' + skipMetricValidation: true + criterionType: 'StaticThresholdCriterion' + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + } +} + +resource Container_working_set_memory_usage_violates_the_configured_threshold_for_clustername_CI_20 'microsoft.insights/metricAlerts@2018-03-01' = { + name: 'Container working set memory usage violates the configured threshold for ${clusterName} CI-20' + location: 'global' + properties: { + description: 'This alert monitors container working set memory usage. It uses the threshold defined in the config map.' + severity: alertSeverityNumber + enabled: metricAlertsEnabled + scopes: [ + AksResourceId + ] + evaluationFrequency: evalFrequency + windowSize: windowSize + criteria: { + allOf: [ + { + threshold: 0 + name: 'Metric1' + metricNamespace: 'Insights.Container/containers' + metricName: 'memoryWorkingSetThresholdViolated' + dimensions: [ + { + name: 'controllerName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetes namespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + operator: 'GreaterThan' + timeAggregation: 'Average' + skipMetricValidation: true + criterionType: 'StaticThresholdCriterion' + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + } +} + + +resource PV_usage_violates_the_configured_threshold_for_clustername_CI_21 'microsoft.insights/metricAlerts@2018-03-01' = { + name: 'PV usage violates the configured threshold for ${clusterName} CI-21' + location: 'global' + properties: { + description: 'This alert monitors PV usage. It uses the threshold defined in the config map.' + severity: alertSeverityNumber + enabled: metricAlertsEnabled + scopes: [ + AksResourceId + ] + evaluationFrequency: evalFrequency + windowSize: windowSize + criteria: { + allOf: [ + { + threshold: 0 + name: 'Metric1' + metricNamespace: 'Insights.Container/persistentvolumes' + metricName: 'pvUsageThresholdViolated' + dimensions: [ + { + name: 'podName' + operator: 'Include' + values: [ + '*' + ] + } + { + name: 'kubernetesNamespace' + operator: 'Include' + values: [ + '*' + ] + } + ] + operator: 'GreaterThan' + timeAggregation: 'Average' + skipMetricValidation: true + criterionType: 'StaticThresholdCriterion' + } + ] + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + } + } +} + + +resource Daily_law_datacap 'Microsoft.Insights/scheduledQueryRules@2022-06-15' = { + name: 'Daily data cap breached for workspace ${logAnalyticsWorkspaceName} CIQ-1' + location: logAnalyticsWorkspaceLocation + properties: { + displayName: 'Daily data cap breached for workspace ${logAnalyticsWorkspaceName} CIQ-1' + description: 'This alert monitors daily data cap defined on a workspace and fires when the daily data cap is breached.' + severity: 1 + enabled: metricAlertsEnabled + evaluationFrequency: evalFrequency + scopes: [ + resourceId('microsoft.operationalinsights/workspaces', logAnalyticsWorkspaceName) + ] + windowSize: windowSize + autoMitigate: false + criteria: { + allOf: [ + { + query: '_LogOperation | where Operation == "Data collection Status" | where Detail contains "OverQuota"' + timeAggregation: 'Count' + operator: 'GreaterThan' + threshold: 0 + failingPeriods: { + numberOfEvaluationPeriods: 1 + minFailingPeriodsToAlert: 1 + } + } + ] + } + muteActionsDuration: 'P1D' + } +} diff --git a/templates/common/infra/bicep/core/host/aks/aksnetcontrib.bicep b/templates/common/infra/bicep/core/host/aks/aksnetcontrib.bicep new file mode 100644 index 00000000000..bae871c898d --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/aksnetcontrib.bicep @@ -0,0 +1,44 @@ +//using a seperate module file as Byo subnet scenario caters from where the subnet is in the another resource group +//name/rg required to new up an existing reference and form a dependency +//principalid required as it needs to be used to establish a unique roleassignment name +param byoAKSSubnetId string +param user_identity_principalId string + +@allowed([ + 'Subnet' + 'Vnet' +]) +param rbacAssignmentScope string = 'Subnet' + +var networkContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') + +var existingAksSubnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[10] : '' +var existingAksVnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[8] : '' + +resource existingvnet 'Microsoft.Network/virtualNetworks@2021-02-01' existing = { + name: existingAksVnetName +} +resource existingAksSubnet 'Microsoft.Network/virtualNetworks/subnets@2020-08-01' existing = { + parent: existingvnet + name: existingAksSubnetName +} + +resource subnetRbac 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = if (rbacAssignmentScope == 'subnet') { + name: guid(user_identity_principalId, networkContributorRole, existingAksSubnetName) + scope: existingAksSubnet + properties: { + roleDefinitionId: networkContributorRole + principalId: user_identity_principalId + principalType: 'ServicePrincipal' + } +} + +resource existingVnetRbac 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = if (rbacAssignmentScope != 'subnet') { + name: guid(user_identity_principalId, networkContributorRole, existingAksVnetName) + scope: existingvnet + properties: { + roleDefinitionId: networkContributorRole + principalId: user_identity_principalId + principalType: 'ServicePrincipal' + } +} diff --git a/templates/common/infra/bicep/core/host/aks/appgw.bicep b/templates/common/infra/bicep/core/host/aks/appgw.bicep new file mode 100644 index 00000000000..82c0e2a250c --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/appgw.bicep @@ -0,0 +1,196 @@ +param resourceName string +param location string +param appGwSubnetId string +param privateIpApplicationGateway string +param availabilityZones array +param userAssignedIdentity string +param workspaceId string +param appGWcount int +param appGWmaxCount int + +var appgwName = 'agw-${resourceName}' +var appgwResourceId = resourceId('Microsoft.Network/applicationGateways', '${appgwName}') + +resource appgwpip 'Microsoft.Network/publicIPAddresses@2020-07-01' = { + name: 'pip-agw-${resourceName}' + location: location + sku: { + name: 'Standard' + } + properties: { + publicIPAllocationMethod: 'Static' + } +} + +var frontendPublicIpConfig = { + properties: { + publicIPAddress: { + id: '${appgwpip.id}' + } + } + name: 'appGatewayFrontendIP' +} + +var frontendPrivateIpConfig = { + properties: { + privateIPAllocationMethod: 'Static' + privateIPAddress: privateIpApplicationGateway + subnet: { + id: appGwSubnetId + } + } + name: 'appGatewayPrivateIP' +} + +var tier = 'WAF_v2' +var appGWsku = union({ + name: tier + tier: tier +}, appGWmaxCount == 0 ? { + capacity: appGWcount +} : {}) + +// ugh, need to create a variable with the app gateway properies, because of the conditional key 'autoscaleConfiguration' +var appgwProperties = union({ + sku: appGWsku + gatewayIPConfigurations: [ + { + name: 'besubnet' + properties: { + subnet: { + id: appGwSubnetId + } + } + } + ] + frontendIPConfigurations: empty(privateIpApplicationGateway) ? array(frontendPublicIpConfig) : concat(array(frontendPublicIpConfig), array(frontendPrivateIpConfig)) + frontendPorts: [ + { + name: 'appGatewayFrontendPort' + properties: { + port: 80 + } + } + ] + backendAddressPools: [ + { + name: 'defaultaddresspool' + } + ] + backendHttpSettingsCollection: [ + { + name: 'defaulthttpsetting' + properties: { + port: 80 + protocol: 'Http' + cookieBasedAffinity: 'Disabled' + requestTimeout: 30 + pickHostNameFromBackendAddress: true + } + } + ] + httpListeners: [ + { + name: 'hlisten' + properties: { + frontendIPConfiguration: { + id: '${appgwResourceId}/frontendIPConfigurations/appGatewayFrontendIP' + } + frontendPort: { + id: '${appgwResourceId}/frontendPorts/appGatewayFrontendPort' + } + protocol: 'Http' + } + } + ] + requestRoutingRules: [ + { + name: 'appGwRoutingRuleName' + properties: { + ruleType: 'Basic' + httpListener: { + id: '${appgwResourceId}/httpListeners/hlisten' + } + backendAddressPool: { + id: '${appgwResourceId}/backendAddressPools/defaultaddresspool' + } + backendHttpSettings: { + id: '${appgwResourceId}/backendHttpSettingsCollection/defaulthttpsetting' + } + } + } + ] +}, appGWmaxCount > 0 ? { + autoscaleConfiguration: { + minCapacity: appGWcount + maxCapacity: appGWmaxCount + } +} : {}) + +var appGwZones = !empty(availabilityZones) ? availabilityZones : [] + +// 'identity' is always set until this is fixed: +// https://github.com/Azure/bicep/issues/387#issuecomment-885671296 +resource appgw 'Microsoft.Network/applicationGateways@2020-07-01' = if (!empty(userAssignedIdentity)) { + name: appgwName + location: location + zones: appGwZones + identity: !empty(userAssignedIdentity) ? { + type: 'UserAssigned' + userAssignedIdentities: { + '${userAssignedIdentity}': {} + } + } : {} + properties: appgwProperties +} + +param agicPrincipleId string +var contributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') +// https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-template#new-service-principal +resource appGwAGICContrib 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: appgw + name: guid(resourceGroup().id, appgwName, 'appgwcont') + properties: { + roleDefinitionId: contributor + principalType: 'ServicePrincipal' + principalId: agicPrincipleId + } +} + +var reader = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') +resource appGwAGICRGReader 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { + scope: resourceGroup() + name: guid(resourceGroup().id, appgwName, 'rgread') + properties: { + roleDefinitionId: reader + principalType: 'ServicePrincipal' + principalId: agicPrincipleId + } +} + +output appgwId string = appgw.id +output ApplicationGatewayName string = appgw.name + +// ------------------------------------------------------------------ AppGW Diagnostics +var diagProperties = { + workspaceId: workspaceId + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] +} +resource appgw_Diag 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = if (!empty(workspaceId)) { + scope: appgw + name: 'appgwDiag' + properties: diagProperties +} diff --git a/templates/common/infra/bicep/core/host/aks/bicepconfig.json b/templates/common/infra/bicep/core/host/aks/bicepconfig.json new file mode 100644 index 00000000000..a4efe888291 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/bicepconfig.json @@ -0,0 +1,55 @@ +{ + "analyzers": { + "core": { + "enabled": true, + "verbose": false, + "rules": { + "no-hardcoded-location" : { + "level": "error" + }, + "no-loc-expr-outside-params" : { + "level": "warning" + }, + "protect-commandtoexecute-secrets" : { + "level": "warning" + }, + "explicit-values-for-loc-params" : { + "level": "warning" + }, + "adminusername-should-not-be-literal": { + "level": "warning" + }, + "no-hardcoded-env-urls": { + "level": "warning" + }, + "no-unnecessary-dependson": { + "level": "warning" + }, + "no-unused-params": { + "level": "warning" + }, + "no-unused-vars": { + "level": "warning" + }, + "outputs-should-not-contain-secrets": { + "level": "warning" + }, + "prefer-interpolation": { + "level": "warning" + }, + "secure-parameter-default": { + "level": "warning" + }, + "simplify-interpolation": { + "level": "error" + }, + "use-stable-vm-image": { + "level": "warning" + }, + "secure-secrets-in-params" : { + "level": "error" + } + } + } + } + } \ No newline at end of file diff --git a/templates/common/infra/bicep/core/host/aks/calcAzFwIp.bicep b/templates/common/infra/bicep/core/host/aks/calcAzFwIp.bicep new file mode 100644 index 00000000000..762cdb66709 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/calcAzFwIp.bicep @@ -0,0 +1,10 @@ +// As per https://github.com/Azure/bicep/issues/2189#issuecomment-815962675 this file is being used as a UDF +// Takes a subnet range and returns the AzFirewall private Ip address + +@description('A subnet address for the Azure Firewall') +param vnetFirewallSubnetAddressPrefix string + +var subnetOctets = split(vnetFirewallSubnetAddressPrefix,'.') +var hostIdOctet = '4' + +output FirewallPrivateIp string = '${subnetOctets[0]}.${subnetOctets[1]}.${subnetOctets[2]}.${hostIdOctet}' diff --git a/templates/common/infra/bicep/core/host/aks/dnsZone.bicep b/templates/common/infra/bicep/core/host/aks/dnsZone.bicep new file mode 100644 index 00000000000..2f678304cdb --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/dnsZone.bicep @@ -0,0 +1,47 @@ +param dnsZoneName string +param principalId string +param isPrivate bool +param vnetId string = '' + +resource dns 'Microsoft.Network/dnsZones@2018-05-01' existing = if (!isPrivate) { + name: dnsZoneName +} + +resource privateDns 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (isPrivate) { + name: dnsZoneName +} + +var DNSZoneContributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314') +resource dnsContributor 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = if (!isPrivate) { + scope: dns + name: guid(dns.id, principalId, DNSZoneContributor) + properties: { + roleDefinitionId: DNSZoneContributor + principalType: 'ServicePrincipal' + principalId: principalId + } +} + +var PrivateDNSZoneContributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b12aa53e-6015-4669-85d0-8515ebb3ae7f') +resource privateDnsContributor 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = if (isPrivate) { + scope: privateDns + name: guid(privateDns.id, principalId, PrivateDNSZoneContributor) + properties: { + roleDefinitionId: PrivateDNSZoneContributor + principalType: 'ServicePrincipal' + principalId: principalId + } +} + +resource dns_vnet_link 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (isPrivate && !empty(vnetId)) { + parent: privateDns + name: 'privatedns' + tags: {} + location: 'global' + properties: { + virtualNetwork: { + id: vnetId + } + registrationEnabled: false + } +} diff --git a/templates/common/infra/bicep/core/host/aks/dnsZoneRbac.bicep b/templates/common/infra/bicep/core/host/aks/dnsZoneRbac.bicep new file mode 100644 index 00000000000..ad4ae4354a5 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/dnsZoneRbac.bicep @@ -0,0 +1,26 @@ +//This module facilitates RBAC assigned to specific DNS Zones. +//The DNS Zone Id is extracted and the scope is set correctly. + +@description('The full Azure resource ID of the DNS zone to use for the AKS cluster') +param dnsZoneId string + +@description('The id of a virtual network to be linked to a PRIVATE DNS Zone') +param vnetId string + +@description('The AAD identity to create the RBAC against') +param principalId string + +var dnsZoneRg = !empty(dnsZoneId) ? split(dnsZoneId, '/')[4] : '' +var dnsZoneName = !empty(dnsZoneId) ? split(dnsZoneId, '/')[8] : '' +var isDnsZonePrivate = !empty(dnsZoneId) ? split(dnsZoneId, '/')[7] == 'privateDnsZones' : false + +module dnsZone './dnsZone.bicep' = if (!empty(dnsZoneId)) { + name: 'dns-${dnsZoneName}' + scope: resourceGroup(dnsZoneRg) + params: { + dnsZoneName: dnsZoneName + isPrivate: isDnsZonePrivate + vnetId : vnetId + principalId: principalId + } +} diff --git a/templates/common/infra/bicep/core/host/aks/firewall.bicep b/templates/common/infra/bicep/core/host/aks/firewall.bicep new file mode 100644 index 00000000000..2fd934ce6f6 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/firewall.bicep @@ -0,0 +1,324 @@ +param resourceName string +param location string = resourceGroup().location +param workspaceDiagsId string = '' +param fwSubnetId string +param fwManagementSubnetId string = '' +param vnetAksSubnetAddressPrefix string +param certManagerFW bool = false +param acrPrivatePool bool = false +param acrAgentPoolSubnetAddressPrefix string = '' +param availabilityZones array = [] +param fwSku string + +var firewallPublicIpName = 'pip-afw-${resourceName}' +var firewallManagementPublicIpName = 'pip-mgmt-afw-${resourceName}' + +var managementIpConfig = { + name: 'MgmtIpConf' + properties: { + publicIPAddress: { + id: !empty(fwManagementSubnetId) ? fwManagementIp_pip.id : null + } + subnet:{ + id: !empty(fwManagementSubnetId) ? fwManagementSubnetId : null + } + } +} + +resource fw_pip 'Microsoft.Network/publicIPAddresses@2021-03-01' = { + name: firewallPublicIpName + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + } +} + +resource fwManagementIp_pip 'Microsoft.Network/publicIPAddresses@2021-03-01' = if(fwSku=='Basic') { + name: firewallManagementPublicIpName + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + } +} + +resource fwDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceDiagsId)) { + scope: fw + name: 'fwDiags' + properties: { + workspaceId: workspaceDiagsId + logs: [ + { + category: 'AzureFirewallApplicationRule' + enabled: true + retentionPolicy: { + days: 10 + enabled: false + } + } + { + category: 'AzureFirewallNetworkRule' + enabled: true + retentionPolicy: { + days: 10 + enabled: false + } + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + retentionPolicy: { + enabled: false + days: 0 + } + } + ] + } +} + +@description('Whitelist dnsZone name (required by cert-manager validation process)') +param appDnsZoneName string = '' + +var fw_name = 'afw-${resourceName}' +resource fw 'Microsoft.Network/azureFirewalls@2022-01-01' = { + name: fw_name + location: location + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + sku: { + tier: fwSku + } + ipConfigurations: [ + { + name: 'IpConf1' + properties: { + subnet: { + id: fwSubnetId + } + publicIPAddress: { + id: fw_pip.id + } + } + } + ] + managementIpConfiguration: !empty(fwManagementSubnetId) ? managementIpConfig : null + threatIntelMode: 'Alert' + firewallPolicy: { + id: fwPolicy.id + } + applicationRuleCollections: [] + networkRuleCollections: [] + } +} + +resource fwPolicy 'Microsoft.Network/firewallPolicies@2022-01-01' = { + name: 'afwp-${resourceName}' + location: location + properties: { + sku: { + tier: fwSku + } + threatIntelMode: 'Alert' + threatIntelWhitelist: { + fqdns: [] + ipAddresses: [] + } + } +} + +resource fwpRules 'Microsoft.Network/firewallPolicies/ruleCollectionGroups@2022-01-01' = { + parent: fwPolicy + name: 'AKSConstructionRuleGroup' + properties: { + priority: 200 + ruleCollections: [ + { + ruleCollectionType: 'FirewallPolicyFilterRuleCollection' + name: 'CoreAksNetEgress' + priority: 100 + action: { + type: 'Allow' + } + rules: concat([ + { + name: 'ControlPlaneTCP' + ruleType: 'NetworkRule' + ipProtocols: [ + 'TCP' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + destinationAddresses: [ + 'AzureCloud.${location}' + ] + destinationPorts: [ + '9000' /* For tunneled secure communication between the nodes and the control plane. */ + '22' + ] + } + { + name: 'ControlPlaneUDP' + ruleType: 'NetworkRule' + ipProtocols: [ + 'UDP' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + destinationAddresses: [ + 'AzureCloud.${location}' + ] + destinationPorts: [ + '1194' /* For tunneled secure communication between the nodes and the control plane. */ + ] + } + { + name: 'AzureMonitorForContainers' + ruleType: 'NetworkRule' + ipProtocols: [ + 'TCP' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + destinationAddresses: [ + 'AzureMonitor' + ] + destinationPorts: [ + '443' + ] + } + ], acrPrivatePool ? [ + { + name: 'acr-agentpool' + ruleType: 'NetworkRule' + ipProtocols: [ + 'TCP' + ] + sourceAddresses: [ + acrAgentPoolSubnetAddressPrefix + ] + destinationAddresses: [ + 'AzureKeyVault' + 'Storage' + 'EventHub' + 'AzureActiveDirectory' + 'AzureMonitor' + ] + destinationPorts: [ + '443' + ] + } + ]:[]) + } + { + ruleCollectionType: 'FirewallPolicyFilterRuleCollection' + name: 'CoreAksHttpEgress' + priority: 400 + action: { + type: 'Allow' + } + rules: concat([ + { + name: 'aks' + ruleType: 'ApplicationRule' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + { + port: 80 + protocolType: 'Http' + } + ] + targetFqdns: [] + fqdnTags: [ + 'AzureKubernetesService' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + } + ], certManagerFW ? [ + { + name: 'certman-quay' + ruleType: 'ApplicationRule' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + { + port: 80 + protocolType: 'Http' + } + ] + targetFqdns: [ + 'quay.io' + '*.quay.io' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + } + { + name: 'certman-letsencrypt' + ruleType: 'ApplicationRule' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + { + port: 80 + protocolType: 'Http' + } + ] + targetFqdns: [ + 'letsencrypt.org' + '*.letsencrypt.org' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + } + ] : [], certManagerFW && !empty(appDnsZoneName) ? [ + { + name: 'certman-appDnsZoneName' + ruleType: 'ApplicationRule' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + { + port: 80 + protocolType: 'Http' + } + ] + targetFqdns: [ + appDnsZoneName + '*.${appDnsZoneName}' + ] + sourceAddresses: [ + vnetAksSubnetAddressPrefix + ] + } + ] : []) + } + ] + } +} diff --git a/templates/common/infra/bicep/core/host/aks/keyvault.bicep b/templates/common/infra/bicep/core/host/aks/keyvault.bicep new file mode 100644 index 00000000000..3a6283f3d8e --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/keyvault.bicep @@ -0,0 +1,80 @@ +@minLength(2) +@description('The location to use for the deployment. defaults to Resource Groups location.') +param location string = resourceGroup().location + +@minLength(3) +@maxLength(20) +@description('Used to name all resources') +param resourceName string + +@description('Enable support for private links') +param privateLinks bool = false + +@description('If soft delete protection is enabled') +param keyVaultSoftDelete bool = true + +@description('If purge protection is enabled') +param keyVaultPurgeProtection bool = true + +@description('Add IP to KV firewall allow-list') +param keyVaultIPAllowlist array = [] + +param logAnalyticsWorkspaceId string = '' + +var akvRawName = 'kv-${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' +var akvName = length(akvRawName) > 24 ? substring(akvRawName, 0, 24) : akvRawName + +var kvIPRules = [for kvIp in keyVaultIPAllowlist: { + value: kvIp +}] + +resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' = { + name: akvName + location: location + properties: { + tenantId: subscription().tenantId + sku: { + family: 'A' + name: 'standard' + } + // publicNetworkAccess: whether the vault will accept traffic from public internet. If set to 'disabled' all traffic except private endpoint traffic and that that originates from trusted services will be blocked. + publicNetworkAccess: privateLinks && empty(keyVaultIPAllowlist) ? 'disabled' : 'enabled' + + networkAcls: privateLinks && !empty(keyVaultIPAllowlist) ? { + bypass: 'AzureServices' + defaultAction: 'Deny' + ipRules: kvIPRules + virtualNetworkRules: [] + } : {} + + enableRbacAuthorization: true + enabledForDeployment: false + enabledForDiskEncryption: false + enabledForTemplateDeployment: false + enableSoftDelete: keyVaultSoftDelete + enablePurgeProtection: keyVaultPurgeProtection ? true : json('null') + } +} + +resource kvDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(logAnalyticsWorkspaceId)) { + name: 'kvDiags' + scope: kv + properties: { + workspaceId: logAnalyticsWorkspaceId + logs: [ + { + category: 'AuditEvent' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output keyVaultName string = kv.name +output keyVaultId string = kv.id diff --git a/templates/common/infra/bicep/core/host/aks/keyvaultkey.bicep b/templates/common/infra/bicep/core/host/aks/keyvaultkey.bicep new file mode 100644 index 00000000000..f00bf67017a --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/keyvaultkey.bicep @@ -0,0 +1,24 @@ +param keyVaultName string + +resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' existing = { + name: keyVaultName +} + +resource kvKmsKey 'Microsoft.KeyVault/vaults/keys@2021-11-01-preview' = { + name: 'kmskey' + parent: kv + properties: { + kty: 'RSA' + keySize: 2048 + keyOps: [ + 'wrapKey' + 'unwrapKey' + 'decrypt' + 'encrypt' + 'verify' + 'sign' + ] + } +} + +output keyVaultKmsKeyUri string = kvKmsKey.properties.keyUriWithVersion diff --git a/templates/common/infra/bicep/core/host/aks/keyvaultrbac.bicep b/templates/common/infra/bicep/core/host/aks/keyvaultrbac.bicep new file mode 100644 index 00000000000..6ef65dd1e08 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/keyvaultrbac.bicep @@ -0,0 +1,173 @@ +param keyVaultName string + +@description('An array of Service Principal IDs') +#disable-next-line secure-secrets-in-params //Disabling validation of this linter rule as param does not contain a secret. +param rbacSecretUserSps array = [] + +@description('An array of Service Principal IDs') +#disable-next-line secure-secrets-in-params //Disabling validation of this linter rule as param does not contain a secret. +param rbacSecretOfficerSps array = [] + +@description('An array of Service Principal IDs') +param rbacCertOfficerSps array = [] + +@description('An array of Service Principal IDs') +param rbacCryptoUserSps array = [] + +@description('An array of Service Principal IDs') +param rbacCryptoOfficerSps array = [] + +@description('An array of Service Principal IDs') +param rbacCryptoServiceEncryptSps array = [] + +@description('An array of Service Principal IDs') +param rbacKvContributorSps array = [] + +@description('An array of Service Principal IDs') +param rbacAdminSps array = [] + +@description('An array of User IDs') +param rbacCryptoOfficerUsers array = [] + +@description('An array of User IDs') +#disable-next-line secure-secrets-in-params //Disabling validation of this linter rule as param does not contain a secret. +param rbacSecretOfficerUsers array = [] + +@description('An array of User IDs') +param rbacCertOfficerUsers array = [] + +@description('An array of User IDs') +param rbacAdminUsers array = [] + +var keyVaultAdministratorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '00482a5a-887f-4fb3-b363-3b7fe8e74483') +var keyVaultContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f25e0fa2-a7c8-4377-a976-54943a77a395') +var keyVaultSecretsUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') +var keyVaultSecretsOfficerRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7') +var keyVaultCertsOfficerRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a4417e6f-fecd-4de8-b567-7b0420556985') +var keyVaultCryptoUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') +var keyVaultCryptoOfficerRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '14b46e9e-c2b7-41b4-b07b-48a6ebf60603') +var keyVaultCryptoServiceEncrpytionRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions','e147488a-f6f5-4113-8e2d-b22465e65bf6') + +resource kv 'Microsoft.KeyVault/vaults@2021-11-01-preview' existing = { + name: keyVaultName +} + +resource rbacSecretUserSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacSecretUserSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultSecretsUserRole) + properties: { + roleDefinitionId: keyVaultSecretsUserRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacSecretOfficerSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacSecretOfficerSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultSecretsOfficerRole) + properties: { + roleDefinitionId: keyVaultSecretsOfficerRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacCertsOfficerSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCertOfficerSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCertsOfficerRole) + properties: { + roleDefinitionId: keyVaultCertsOfficerRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacCryptoUserSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCryptoUserSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCryptoUserRole) + properties: { + roleDefinitionId: keyVaultCryptoUserRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacCryptoServiceEncryptionSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCryptoServiceEncryptSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCryptoServiceEncrpytionRole) + properties: { + roleDefinitionId: keyVaultCryptoServiceEncrpytionRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacKvContributorSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacKvContributorSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultContributorRole) + properties: { + roleDefinitionId: keyVaultContributorRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacCryptoOfficerSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCryptoOfficerSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCryptoUserRole) + properties: { + roleDefinitionId: keyVaultCryptoOfficerRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacAdminSp 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacAdminSps : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultAdministratorRole) + properties: { + roleDefinitionId: keyVaultAdministratorRole + principalType: 'ServicePrincipal' + principalId: rbacSp + } +}] + +resource rbacCryptoOfficerUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCryptoOfficerUsers : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCryptoOfficerRole) + properties: { + roleDefinitionId: keyVaultCryptoOfficerRole + principalType: 'User' + principalId: rbacSp + } +}] + +resource rbacSecretOfficerUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacSecretOfficerUsers : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultSecretsOfficerRole) + properties: { + roleDefinitionId: keyVaultSecretsOfficerRole + principalType: 'User' + principalId: rbacSp + } +}] + +resource rbacCertsOfficerUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacCertOfficerUsers : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultCertsOfficerRole) + properties: { + roleDefinitionId: keyVaultCertsOfficerRole + principalType: 'User' + principalId: rbacSp + } +}] + +resource rbacAdminUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for rbacSp in rbacAdminUsers : if(!empty(rbacSp)) { + scope: kv + name: guid(kv.id, rbacSp, keyVaultAdministratorRole) + properties: { + roleDefinitionId: keyVaultAdministratorRole + principalType: 'User' + principalId: rbacSp + } +}] diff --git a/templates/common/infra/bicep/core/host/aks/main.bicep b/templates/common/infra/bicep/core/host/aks/main.bicep new file mode 100644 index 00000000000..47553d34e04 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/main.bicep @@ -0,0 +1,1627 @@ +@minLength(2) +@description('The location to use for the deployment. defaults to Resource Groups location.') +param location string = resourceGroup().location + +@minLength(3) +@maxLength(20) +@description('Used to name all resources') +param resourceName string + +/* +Resource sections +1. Networking +2. DNS +3. Key Vault +4. Container Registry +5. Firewall +6. Application Gateway +7. AKS +8. Monitoring / Log Analytics +9. Deployment for telemetry +*/ + + +/*.__ __. _______ .___________.____ __ ____ ______ .______ __ ___ __ .__ __. _______ +| \ | | | ____|| |\ \ / \ / / / __ \ | _ \ | |/ / | | | \ | | / _____| +| \| | | |__ `---| |----` \ \/ \/ / | | | | | |_) | | ' / | | | \| | | | __ +| . ` | | __| | | \ / | | | | | / | < | | | . ` | | | |_ | +| |\ | | |____ | | \ /\ / | `--' | | |\ \----.| . \ | | | |\ | | |__| | +|__| \__| |_______| |__| \__/ \__/ \______/ | _| `._____||__|\__\ |__| |__| \__| \______| */ +//Networking can either be one of: custom / byo / default + +@description('Are you providing your own vNet CIDR blocks') +param custom_vnet bool = false + +@description('Full resource id path of an existing subnet to use for AKS') +param byoAKSSubnetId string = '' + +@description('Full resource id path of an existing subnet to use for Application Gateway') +param byoAGWSubnetId string = '' + +//--- Custom, BYO networking and PrivateApiZones requires BYO AKS User Identity +var createAksUai = custom_vnet || !empty(byoAKSSubnetId) || !empty(dnsApiPrivateZoneId) || keyVaultKmsCreateAndPrereqs || !empty(keyVaultKmsByoKeyId) +resource aksUai 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = if (createAksUai) { + name: 'id-aks-${resourceName}' + location: location +} + +//----------------------------------------------------- BYO Vnet +var existingAksVnetRG = !empty(byoAKSSubnetId) ? (length(split(byoAKSSubnetId, '/')) > 4 ? split(byoAKSSubnetId, '/')[4] : '') : '' + +module aksnetcontrib './aksnetcontrib.bicep' = if (!empty(byoAKSSubnetId) && createAksUai) { + name: 'addAksNetContributor' + scope: resourceGroup(existingAksVnetRG) + params: { + byoAKSSubnetId: byoAKSSubnetId + user_identity_principalId: createAksUai ? aksUai.properties.principalId : '' + rbacAssignmentScope: uaiNetworkScopeRbac + } +} + +//------------------------------------------------------ Create custom vnet +@minLength(9) +@maxLength(18) +@description('The address range for the custom vnet') +param vnetAddressPrefix string = '10.240.0.0/16' + +@minLength(9) +@maxLength(18) +@description('The address range for AKS in your custom vnet') +param vnetAksSubnetAddressPrefix string = '10.240.0.0/22' + +@minLength(9) +@maxLength(18) +@description('The address range for the App Gateway in your custom vnet') +param vnetAppGatewaySubnetAddressPrefix string = '10.240.5.0/24' + +@minLength(9) +@maxLength(18) +@description('The address range for the ACR in your custom vnet') +param acrAgentPoolSubnetAddressPrefix string = '10.240.4.64/26' + +@minLength(9) +@maxLength(18) +@description('The address range for Azure Bastion in your custom vnet') +param bastionSubnetAddressPrefix string = '10.240.4.128/26' + +@minLength(9) +@maxLength(18) +@description('The address range for private link in your custom vnet') +param privateLinkSubnetAddressPrefix string = '10.240.4.192/26' + +@minLength(9) +@maxLength(18) +@description('The address range for Azure Firewall in your custom vnet') +param vnetFirewallSubnetAddressPrefix string = '10.240.50.0/24' + +@minLength(9) +@maxLength(18) +@description('The address range for Azure Firewall Management in your custom vnet') +param vnetFirewallManagementSubnetAddressPrefix string = '10.240.51.0/26' + +@description('Enable support for private links (required custom_vnet)') +param privateLinks bool = false + +@description('Enable support for ACR private pool') +param acrPrivatePool bool = false + +@description('Deploy Azure Bastion to your vnet. (works with Custom Networking only, not BYO)') +param bastion bool = false + +@description('Deploy NSGs to your vnet subnets. (works with Custom Networking only, not BYO)') +param CreateNetworkSecurityGroups bool = false + +@description('Configure Flow Logs for Network Security Groups in the NetworkWatcherRG resource group. Requires Contributor RBAC on NetworkWatcherRG and Reader on Subscription.') +param CreateNetworkSecurityGroupFlowLogs bool = false + +module network './network.bicep' = if (custom_vnet) { + name: 'network' + params: { + resourceName: resourceName + location: location + networkPluginIsKubenet: networkPlugin=='kubenet' + vnetAddressPrefix: vnetAddressPrefix + aksPrincipleId: createAksUai ? aksUai.properties.principalId : '' + vnetAksSubnetAddressPrefix: vnetAksSubnetAddressPrefix + ingressApplicationGateway: ingressApplicationGateway + vnetAppGatewaySubnetAddressPrefix: vnetAppGatewaySubnetAddressPrefix + azureFirewalls: azureFirewalls + vnetFirewallSubnetAddressPrefix: vnetFirewallSubnetAddressPrefix + vnetFirewallManagementSubnetAddressPrefix: vnetFirewallManagementSubnetAddressPrefix + privateLinks: privateLinks + privateLinkSubnetAddressPrefix: privateLinkSubnetAddressPrefix + privateLinkAcrId: privateLinks && !empty(registries_sku) ? acr.id : '' + privateLinkAkvId: privateLinks && keyVaultCreate ? kv.outputs.keyVaultId : '' + acrPrivatePool: acrPrivatePool + acrAgentPoolSubnetAddressPrefix: acrAgentPoolSubnetAddressPrefix + bastion: bastion + bastionSubnetAddressPrefix: bastionSubnetAddressPrefix + availabilityZones: availabilityZones + workspaceName: createLaw ? aks_law.name : '' + workspaceResourceGroupName: createLaw ? resourceGroup().name : '' + networkSecurityGroups: CreateNetworkSecurityGroups + CreateNsgFlowLogs: CreateNetworkSecurityGroups && CreateNetworkSecurityGroupFlowLogs + ingressApplicationGatewayPublic: empty(privateIpApplicationGateway) + natGateway: createNatGateway + natGatewayIdleTimeoutMins: natGwIdleTimeout + natGatewayPublicIps: natGwIpCount + } +} +output CustomVnetId string = custom_vnet ? network.outputs.vnetId : '' +output CustomVnetPrivateLinkSubnetId string = custom_vnet ? network.outputs.privateLinkSubnetId : '' + +var aksSubnetId = custom_vnet ? network.outputs.aksSubnetId : byoAKSSubnetId +var appGwSubnetId = ingressApplicationGateway ? (custom_vnet ? network.outputs.appGwSubnetId : byoAGWSubnetId) : '' + + +/*______ .__ __. _______. ________ ______ .__ __. _______ _______. +| \ | \ | | / | | / / __ \ | \ | | | ____| / | +| .--. || \| | | (----` `---/ / | | | | | \| | | |__ | (----` +| | | || . ` | \ \ / / | | | | | . ` | | __| \ \ +| '--' || |\ | .----) | / /----.| `--' | | |\ | | |____ .----) | +|_______/ |__| \__| |_______/ /________| \______/ |__| \__| |_______||_______/ */ + +@description('The full Azure resource ID of the DNS zone to use for the AKS cluster') +param dnsZoneId string = '' +var isDnsZonePrivate = !empty(dnsZoneId) ? split(dnsZoneId, '/')[7] == 'privateDnsZones' : false + +module dnsZone './dnsZoneRbac.bicep' = if (!empty(dnsZoneId)) { + name: 'addDnsContributor' + params: { + dnsZoneId: dnsZoneId + vnetId: isDnsZonePrivate ? (!empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/subnets')[0] : (custom_vnet ? network.outputs.vnetId : '')) : '' + principalId: any(aks.properties.identityProfile.kubeletidentity).objectId + } +} + + +/*__ __ _______ ____ ____ ____ ____ ___ __ __ __ .___________. +| |/ / | ____|\ \ / / \ \ / / / \ | | | | | | | | +| ' / | |__ \ \/ / \ \/ / / ^ \ | | | | | | `---| |----` +| < | __| \_ _/ \ / / /_\ \ | | | | | | | | +| . \ | |____ | | \ / / _____ \ | `--' | | `----. | | +|__|\__\ |_______| |__| \__/ /__/ \__\ \______/ |_______| |__| */ + +@description('Creates a KeyVault') +param keyVaultCreate bool = false + +@description('If soft delete protection is enabled') +param keyVaultSoftDelete bool = true + +@description('If purge protection is enabled') +param keyVaultPurgeProtection bool = true + +@description('Add IP to KV firewall allow-list') +param keyVaultIPAllowlist array = [] + +@description('Installs the AKS KV CSI provider') +param keyVaultAksCSI bool = false + +@description('Rotation poll interval for the AKS KV CSI provider') +param keyVaultAksCSIPollInterval string = '2m' + +@description('Creates a KeyVault for application secrets (eg. CSI)') +module kv 'keyvault.bicep' = if(keyVaultCreate) { + name: 'keyvaultApps' + params: { + resourceName: resourceName + keyVaultPurgeProtection: keyVaultPurgeProtection + keyVaultSoftDelete: keyVaultSoftDelete + keyVaultIPAllowlist: keyVaultIPAllowlist + location: location + privateLinks: privateLinks + } +} + +@description('The principal ID of the user or service principal that requires access to the Key Vault. Set automatedDeployment to toggle between user and service prinicpal') +param keyVaultOfficerRolePrincipalId string = '' +var keyVaultOfficerRolePrincipalIds = [ + keyVaultOfficerRolePrincipalId +] + +@description('Parsing an array with union ensures that duplicates are removed, which is great when dealing with highly conditional elements') +var rbacSecretUserSps = union([deployAppGw && appgwKVIntegration ? appGwIdentity.properties.principalId : ''],[keyVaultAksCSI ? aks.properties.addonProfiles.azureKeyvaultSecretsProvider.identity.objectId : '']) + +@description('A seperate module is used for RBAC to avoid delaying the KeyVault creation and causing a circular reference.') +module kvRbac 'keyvaultrbac.bicep' = if (keyVaultCreate) { + name: 'KeyVaultAppsRbac' + params: { + keyVaultName: keyVaultCreate ? kv.outputs.keyVaultName : '' + + //service principals + rbacSecretUserSps: rbacSecretUserSps + rbacSecretOfficerSps: !empty(keyVaultOfficerRolePrincipalId) && automatedDeployment ? keyVaultOfficerRolePrincipalIds : [] + rbacCertOfficerSps: !empty(keyVaultOfficerRolePrincipalId) && automatedDeployment ? keyVaultOfficerRolePrincipalIds : [] + + //users + rbacSecretOfficerUsers: !empty(keyVaultOfficerRolePrincipalId) && !automatedDeployment ? keyVaultOfficerRolePrincipalIds : [] + rbacCertOfficerUsers: !empty(keyVaultOfficerRolePrincipalId) && !automatedDeployment ? keyVaultOfficerRolePrincipalIds : [] + } +} + +output keyVaultName string = keyVaultCreate ? kv.outputs.keyVaultName : '' +output keyVaultId string = keyVaultCreate ? kv.outputs.keyVaultId : '' + + +/* KeyVault for KMS Etcd*/ + +@description('Enable encryption at rest for Kubernetes etcd data') +param keyVaultKmsCreate bool = false + +@description('Bring an existing Key from an existing Key Vault') +param keyVaultKmsByoKeyId string = '' + +@description('The resource group for the existing KMS Key Vault') +param keyVaultKmsByoRG string = resourceGroup().name + +@description('The PrincipalId of the deploying user, which is necessary when creating a Kms Key') +param keyVaultKmsOfficerRolePrincipalId string = '' + +@description('The extracted name of the existing Key Vault') +var keyVaultKmsByoName = !empty(keyVaultKmsByoKeyId) ? split(split(keyVaultKmsByoKeyId,'/')[2],'.')[0] : '' + +@description('The deployment delay to introduce when creating a new keyvault for kms key') +var kmsRbacWaitSeconds=30 + +@description('This indicates if the deploying user has provided their PrincipalId in order for the key to be created') +var keyVaultKmsCreateAndPrereqs = keyVaultKmsCreate && !empty(keyVaultKmsOfficerRolePrincipalId) && privateLinks == false + +resource kvKmsByo 'Microsoft.KeyVault/vaults@2021-11-01-preview' existing = if(!empty(keyVaultKmsByoName)) { + name: keyVaultKmsByoName + scope: resourceGroup(keyVaultKmsByoRG) +} + +@description('Creates a new Key vault for a new KMS Key') +module kvKms 'keyvault.bicep' = if(keyVaultKmsCreateAndPrereqs) { + name: 'keyvaultKms-${resourceName}' + params: { + resourceName: 'kms${resourceName}' + keyVaultPurgeProtection: keyVaultPurgeProtection + keyVaultSoftDelete: keyVaultSoftDelete + location: location + privateLinks: privateLinks + //aksUaiObjectId: aksUai.properties.principalId + } +} + +module kvKmsCreatedRbac 'keyvaultrbac.bicep' = if(keyVaultKmsCreateAndPrereqs) { + name: 'keyvaultKmsRbacs-${resourceName}' + params: { + keyVaultName: keyVaultKmsCreate ? kvKms.outputs.keyVaultName : '' + //We can't create a kms kv and key and do privatelink. Private Link is a BYO scenario + // rbacKvContributorSps : [ + // createAksUai && privateLinks ? aksUai.properties.principalId : '' + // ] + //This allows the Aks Cluster to access the key vault key + rbacCryptoUserSps: [ + createAksUai ? aksUai.properties.principalId : '' + ] + //This allows the Deploying user to create the key vault key + rbacCryptoOfficerUsers: [ + !empty(keyVaultKmsOfficerRolePrincipalId) && !automatedDeployment ? keyVaultKmsOfficerRolePrincipalId : '' + ] + //This allows the Deploying sp to create the key vault key + rbacCryptoOfficerSps: [ + !empty(keyVaultKmsOfficerRolePrincipalId) && automatedDeployment ? keyVaultKmsOfficerRolePrincipalId : '' + ] + } +} + +module kvKmsByoRbac 'keyvaultrbac.bicep' = if(!empty(keyVaultKmsByoKeyId)) { + name: 'keyvaultKmsByoRbacs-${resourceName}' + scope: resourceGroup(keyVaultKmsByoRG) + params: { + keyVaultName: kvKmsByo.name + //Contribuor allows AKS to create the private link + rbacKvContributorSps : [ + createAksUai && privateLinks ? aksUai.properties.principalId : '' + ] + //This allows the Aks Cluster to access the key vault key + rbacCryptoUserSps: [ + createAksUai ? aksUai.properties.principalId : '' + ] + } +} + +@description('It can take time for the RBAC to propagate, this delays the deployment to avoid this problem') +module waitForKmsRbac 'br/public:deployment-scripts/wait:1.0.1' = if(keyVaultKmsCreateAndPrereqs && kmsRbacWaitSeconds>0) { + name: 'keyvaultKmsRbac-waits-${resourceName}' + params: { + waitSeconds: kmsRbacWaitSeconds + location: location + } + dependsOn: [ + kvKmsCreatedRbac + ] +} + +@description('Adding a key to the keyvault... We can only do this for public key vaults') +module kvKmsKey 'keyvaultkey.bicep' = if(keyVaultKmsCreateAndPrereqs) { + name: 'keyvaultKmsKeys-${resourceName}' + params: { + keyVaultName: keyVaultKmsCreateAndPrereqs ? kvKms.outputs.keyVaultName : '' + } + dependsOn: [waitForKmsRbac] +} + +var azureKeyVaultKms = { + securityProfile : { + azureKeyVaultKms : { + enabled: keyVaultKmsCreateAndPrereqs || !empty(keyVaultKmsByoKeyId) + keyId: keyVaultKmsCreateAndPrereqs ? kvKmsKey.outputs.keyVaultKmsKeyUri : !empty(keyVaultKmsByoKeyId) ? keyVaultKmsByoKeyId : '' + keyVaultNetworkAccess: privateLinks ? 'private' : 'public' + keyVaultResourceId: privateLinks && !empty(keyVaultKmsByoKeyId) ? kvKmsByo.id : '' + } + } +} + +@description('The name of the Kms Key Vault') +output keyVaultKmsName string = keyVaultKmsCreateAndPrereqs ? kvKms.outputs.keyVaultName : !empty(keyVaultKmsByoKeyId) ? keyVaultKmsByoName : '' + +@description('Indicates if the user has supplied all the correct parameter to use a AKSC Created KMS') +output kmsCreatePrerequisitesMet bool = keyVaultKmsCreateAndPrereqs + + +/* ___ ______ .______ + / \ / | | _ \ + / ^ \ | ,----' | |_) | + / /_\ \ | | | / + / _____ \ __| `----. __ | |\ \----. __ +/__/ \__\ (__)\______|(__)| _| `._____|(__)*/ + +@allowed([ + '' + 'Basic' + 'Standard' + 'Premium' +]) +@description('The SKU to use for the Container Registry') +param registries_sku string = '' + +@description('Enable the ACR Content Trust Policy, SKU must be set to Premium') +param enableACRTrustPolicy bool = false +var acrContentTrustEnabled = enableACRTrustPolicy && registries_sku == 'Premium' ? 'enabled' : 'disabled' + +//param enableACRZoneRedundancy bool = true +var acrZoneRedundancyEnabled = !empty(availabilityZones) && registries_sku == 'Premium' ? 'Enabled' : 'Disabled' + +@description('Enable removing of untagged manifests from ACR') +param acrUntaggedRetentionPolicyEnabled bool = false + +@description('The number of days to retain untagged manifests for') +param acrUntaggedRetentionPolicy int = 30 + +var acrName = 'cr${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' + +resource acr 'Microsoft.ContainerRegistry/registries@2021-06-01-preview' = if (!empty(registries_sku)) { + name: acrName + location: location + sku: { + #disable-next-line BCP036 //Disabling validation of this parameter to cope with empty string to indicate no ACR required. + name: registries_sku + } + properties: { + policies: { + trustPolicy: enableACRTrustPolicy ? { + status: acrContentTrustEnabled + type: 'Notary' + } : {} + retentionPolicy: acrUntaggedRetentionPolicyEnabled ? { + status: 'enabled' + days: acrUntaggedRetentionPolicy + } : json('null') + } + publicNetworkAccess: privateLinks /* && empty(acrIPWhitelist)*/ ? 'Disabled' : 'Enabled' + zoneRedundancy: acrZoneRedundancyEnabled + /* + networkRuleSet: { + defaultAction: 'Deny' + + ipRules: empty(acrIPWhitelist) ? [] : [ + { + action: 'Allow' + value: acrIPWhitelist + } + ] + virtualNetworkRules: [] + } + */ + } +} +output containerRegistryName string = !empty(registries_sku) ? acr.name : '' +output containerRegistryId string = !empty(registries_sku) ? acr.id : '' + + +resource acrDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (createLaw && !empty(registries_sku)) { + name: 'acrDiags' + scope: acr + properties: { + workspaceId:aks_law.id + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + timeGrain: 'PT1M' + } + ] + } +} + +//resource acrPool 'Microsoft.ContainerRegistry/registries/agentPools@2019-06-01-preview' = if (custom_vnet && (!empty(registries_sku)) && privateLinks && acrPrivatePool) { +module acrPool 'acragentpool.bicep' = if (custom_vnet && (!empty(registries_sku)) && privateLinks && acrPrivatePool) { + name: 'acrprivatepool' + params: { + acrName: acr.name + acrPoolSubnetId: custom_vnet ? network.outputs.acrPoolSubnetId : '' + location: location + } +} + +var AcrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') +var KubeletObjectId = any(aks.properties.identityProfile.kubeletidentity).objectId + +resource aks_acr_pull 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(registries_sku)) { + scope: acr // Use when specifying a scope that is different than the deployment scope + name: guid(aks.id, 'Acr' , AcrPullRole) + properties: { + roleDefinitionId: AcrPullRole + principalType: 'ServicePrincipal' + principalId: KubeletObjectId + } +} + +var AcrPushRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8311e382-0749-4cb8-b61a-304f252e45ec') + +@description('The principal ID of the service principal to assign the push role to the ACR') +param acrPushRolePrincipalId string = '' + +resource aks_acr_push 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(registries_sku) && !empty(acrPushRolePrincipalId)) { + scope: acr // Use when specifying a scope that is different than the deployment scope + name: guid(aks.id, 'Acr' , AcrPushRole) + properties: { + roleDefinitionId: AcrPushRole + principalType: automatedDeployment ? 'ServicePrincipal' : 'User' + principalId: acrPushRolePrincipalId + } +} + + +param imageNames array = [] + +module acrImport 'br/public:deployment-scripts/import-acr:2.0.1' = if (!empty(registries_sku) && !empty(imageNames)) { + name: 'testAcrImportMulti' + params: { + acrName: acr.name + location: location + images: imageNames + } +} + + + + +/*______ __ .______ _______ ____ __ ____ ___ __ __ +| ____|| | | _ \ | ____|\ \ / \ / / / \ | | | | +| |__ | | | |_) | | |__ \ \/ \/ / / ^ \ | | | | +| __| | | | / | __| \ / / /_\ \ | | | | +| | | | | |\ \----.| |____ \ /\ / / _____ \ | `----.| `----. +|__| |__| | _| `._____||_______| \__/ \__/ /__/ \__\ |_______||_______|*/ + +@description('Create an Azure Firewall, requires custom_vnet') +param azureFirewalls bool = false + +@description('Add application rules to the firewall for certificate management.') +param certManagerFW bool = false + +// @allowed([ +// 'AllowAllIn' +// 'AllowAcrSubnetIn' +// '' +// ]) +// @description('Allow Http traffic (80/443) into AKS from specific sources') +// param inboundHttpFW string = '' + +@allowed([ + 'Basic' + 'Premium' + 'Standard' +]) +param azureFirewallSku string = 'Standard' + +module firewall './firewall.bicep' = if (azureFirewalls && custom_vnet) { + name: 'firewall' + params: { + resourceName: resourceName + location: location + workspaceDiagsId: createLaw ? aks_law.id : '' + fwSubnetId: azureFirewalls && custom_vnet ? network.outputs.fwSubnetId : '' + fwSku: azureFirewallSku + fwManagementSubnetId: azureFirewalls && custom_vnet && azureFirewallSku=='Basic' ? network.outputs.fwMgmtSubnetId : '' + vnetAksSubnetAddressPrefix: vnetAksSubnetAddressPrefix + certManagerFW: certManagerFW + appDnsZoneName: !empty(dnsZoneId) ? split(dnsZoneId, '/')[8] : '' + acrPrivatePool: acrPrivatePool + acrAgentPoolSubnetAddressPrefix: acrAgentPoolSubnetAddressPrefix + // inboundHttpFW: inboundHttpFW + availabilityZones: availabilityZones + } +} + +/* ___ .______ .______ _______ ____ __ ____ + / \ | _ \ | _ \ / _____|\ \ / \ / / + / ^ \ | |_) | | |_) | | | __ \ \/ \/ / + / /_\ \ | ___/ | ___/ | | |_ | \ / + / _____ \ | | | | __ | |__| | \ /\ / __ +/__/ \__\ | _| | _| (__) \______| \__/ \__/ (__)*/ + +@description('Create an Application Gateway') +param ingressApplicationGateway bool = false + +@description('The number of applciation gateway instances') +param appGWcount int = 2 + +@description('The maximum number of application gateway instances') +param appGWmaxCount int = 0 + +@maxLength(15) +@description('A known private ip in the Application Gateway subnet range to be allocated for internal traffic') +param privateIpApplicationGateway string = '' + +@description('Enable key vault integration for application gateway') +param appgwKVIntegration bool = false + +@allowed([ + 'Standard_v2' + 'WAF_v2' +]) +@description('The SKU for AppGw') +param appGWsku string = 'WAF_v2' + +@description('Enable the WAF Firewall, valid for WAF_v2 SKUs') +param appGWenableFirewall bool = true + +var deployAppGw = ingressApplicationGateway && (custom_vnet || !empty(byoAGWSubnetId)) +var appGWenableWafFirewall = appGWsku=='Standard_v2' ? false : appGWenableFirewall + +// If integrating App Gateway with KeyVault, create a Identity App Gateway will use to access keyvault +// 'identity' is always created (adding: "|| deployAppGw") until this is fixed: +// https://github.com/Azure/bicep/issues/387#issuecomment-885671296 +resource appGwIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = if (deployAppGw) { + name: 'id-appgw-${resourceName}' + location: location +} + +var appgwName = 'agw-${resourceName}' +var appgwResourceId = deployAppGw ? resourceId('Microsoft.Network/applicationGateways', '${appgwName}') : '' + +resource appgwpip 'Microsoft.Network/publicIPAddresses@2021-02-01' = if (deployAppGw) { + name: 'pip-agw-${resourceName}' + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIPAllocationMethod: 'Static' + } +} + +var frontendPublicIpConfig = { + properties: { + publicIPAddress: { + id: appgwpip.id + } + } + name: 'appGatewayFrontendIP' +} + +var frontendPrivateIpConfig = { + properties: { + privateIPAllocationMethod: 'Static' + privateIPAddress: privateIpApplicationGateway + subnet: { + id: appGwSubnetId + } + } + name: 'appGatewayPrivateIP' +} + +@allowed([ + 'Prevention' + 'Detection' +]) +param appGwFirewallMode string = 'Prevention' + +var appGwFirewallConfigOwasp = { + enabled: appGWenableWafFirewall + firewallMode: appGwFirewallMode + ruleSetType: 'OWASP' + ruleSetVersion: '3.2' + requestBodyCheck: true + maxRequestBodySizeInKb: 128 + disabledRuleGroups: [] +} + +var appGWskuObj = union({ + name: appGWsku + tier: appGWsku +}, appGWmaxCount == 0 ? { + capacity: appGWcount +} : {}) + +// ugh, need to create a variable with the app gateway properies, because of the conditional key 'autoscaleConfiguration' +var appgwProperties = union({ + sku: appGWskuObj + sslPolicy: { + policyType: 'Predefined' + policyName: 'AppGwSslPolicy20170401S' + } + webApplicationFirewallConfiguration: appGWenableWafFirewall ? appGwFirewallConfigOwasp : json('null') + gatewayIPConfigurations: [ + { + name: 'besubnet' + properties: { + subnet: { + id: appGwSubnetId + } + } + } + ] + frontendIPConfigurations: empty(privateIpApplicationGateway) ? array(frontendPublicIpConfig) : concat(array(frontendPublicIpConfig), array(frontendPrivateIpConfig)) + frontendPorts: [ + { + name: 'appGatewayFrontendPort' + properties: { + port: 80 + } + } + ] + backendAddressPools: [ + { + name: 'defaultaddresspool' + } + ] + backendHttpSettingsCollection: [ + { + name: 'defaulthttpsetting' + properties: { + port: 80 + protocol: 'Http' + cookieBasedAffinity: 'Disabled' + requestTimeout: 30 + pickHostNameFromBackendAddress: true + } + } + ] + httpListeners: [ + { + name: 'hlisten' + properties: { + frontendIPConfiguration: { + id: empty(privateIpApplicationGateway) ? '${appgwResourceId}/frontendIPConfigurations/appGatewayFrontendIP' : '${appgwResourceId}/frontendIPConfigurations/appGatewayPrivateIP' + } + frontendPort: { + id: '${appgwResourceId}/frontendPorts/appGatewayFrontendPort' + } + protocol: 'Http' + } + } + ] + requestRoutingRules: [ + { + name: 'appGwRoutingRuleName' + properties: { + ruleType: 'Basic' + httpListener: { + id: '${appgwResourceId}/httpListeners/hlisten' + } + backendAddressPool: { + id: '${appgwResourceId}/backendAddressPools/defaultaddresspool' + } + backendHttpSettings: { + id: '${appgwResourceId}/backendHttpSettingsCollection/defaulthttpsetting' + } + } + } + ] +}, appGWmaxCount > 0 ? { + autoscaleConfiguration: { + minCapacity: appGWcount + maxCapacity: appGWmaxCount + } +} : {}) + +// 'identity' is always set until this is fixed: https://github.com/Azure/bicep/issues/387#issuecomment-885671296 +resource appgw 'Microsoft.Network/applicationGateways@2021-02-01' = if (deployAppGw) { + name: appgwName + location: location + zones: !empty(availabilityZones) ? availabilityZones : [] + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${appGwIdentity.id}': {} + } + } + properties: appgwProperties +} + +var contributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') +// https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-template#new-service-principal +// AGIC's identity requires "Contributor" permission over Application Gateway. +resource appGwAGICContrib 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (ingressApplicationGateway && deployAppGw) { + scope: appgw + name: guid(aks.id, 'Agic', contributor) + properties: { + roleDefinitionId: contributor + principalType: 'ServicePrincipal' + principalId: aks.properties.addonProfiles.ingressApplicationGateway.identity.objectId + } +} + +// AGIC's identity requires "Reader" permission over Application Gateway's resource group. +var reader = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') +resource appGwAGICRGReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (ingressApplicationGateway && deployAppGw) { + scope: resourceGroup() + name: guid(aks.id, 'Agic', reader) + properties: { + roleDefinitionId: reader + principalType: 'ServicePrincipal' + principalId: aks.properties.addonProfiles.ingressApplicationGateway.identity.objectId + } +} + +// AGIC's identity requires "Managed Identity Operator" permission over the user assigned identity of Application Gateway. +var managedIdentityOperator = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') +resource appGwAGICMIOp 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (ingressApplicationGateway && deployAppGw) { + scope: appGwIdentity + name: guid(aks.id, 'Agic', managedIdentityOperator) + properties: { + roleDefinitionId: managedIdentityOperator + principalType: 'ServicePrincipal' + principalId: aks.properties.addonProfiles.ingressApplicationGateway.identity.objectId + } +} + +// AppGW Diagnostics +resource appgw_Diag 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (createLaw && deployAppGw) { + scope: appgw + name: 'appgwDiag' + properties: { + workspaceId: aks_law.id + logs: [ + { + category: 'ApplicationGatewayAccessLog' + enabled: true + } + { + category: 'ApplicationGatewayPerformanceLog' + enabled: true + } + { + category: 'ApplicationGatewayFirewallLog' + enabled: true + } + ] + } +} + +// ================================================= + +// Prevent error: AGIC Identity needs atleast has 'Contributor' access to Application Gateway and 'Reader' access to Application Gateway's Resource Group + +output ApplicationGatewayName string = deployAppGw ? appgw.name : '' + +/*_ ___ __ __ .______ _______ .______ .__ __. _______ .___________. _______ _______. +| |/ / | | | | | _ \ | ____|| _ \ | \ | | | ____|| || ____| / | +| ' / | | | | | |_) | | |__ | |_) | | \| | | |__ `---| |----`| |__ | (----` +| < | | | | | _ < | __| | / | . ` | | __| | | | __| \ \ +| . \ | `--' | | |_) | | |____ | |\ \----.| |\ | | |____ | | | |____ .----) | +|__|\__\ \______/ |______/ |_______|| _| `._____||__| \__| |_______| |__| |_______||_______/ */ + +@description('DNS prefix. Defaults to {resourceName}-dns') +param dnsPrefix string = '${resourceName}-dns' + +@description('Kubernetes Version') +param kubernetesVersion string = '1.23.12' + +@description('Enable Azure AD integration on AKS') +param enable_aad bool = false + +@description('The ID of the Azure AD tenant') +param aad_tenant_id string = '' + +@description('Create, and use a new Log Analytics workspace for AKS logs') +param omsagent bool = false + +@description('Enables the ContainerLogsV2 table to be of type Basic') +param containerLogsV2BasicLogs bool = false + +@description('Enable RBAC using AAD') +param enableAzureRBAC bool = false + +@description('Enables Kubernetes Event-driven Autoscaling (KEDA)') +param kedaAddon bool = false + +@description('Enables Open Service Mesh') +param openServiceMeshAddon bool = false + +@description('Enables the Blob CSI driver') +param blobCSIDriver bool = false + +@description('Enables the File CSI driver') +param fileCSIDriver bool = true + +@description('Enables the Disk CSI driver') +param diskCSIDriver bool = true + +@allowed([ + 'none' + 'patch' + 'stable' + 'rapid' + 'node-image' +]) +@description('AKS upgrade channel') +param upgradeChannel string = 'none' + +@allowed([ + 'Ephemeral' + 'Managed' +]) +@description('OS disk type') +param osDiskType string = 'Ephemeral' + +@description('VM SKU') +param agentVMSize string = 'Standard_DS3_v2' + +@description('Disk size in GB') +param osDiskSizeGB int = 0 + +@description('The number of agents for the user node pool') +param agentCount int = 3 + +@description('The maximum number of nodes for the user node pool') +param agentCountMax int = 0 +var autoScale = agentCountMax > agentCount + +@minLength(3) +@maxLength(12) +@description('Name for user node pool') +param nodePoolName string = 'npuser01' + +@description('Allocate pod ips dynamically') +param cniDynamicIpAllocation bool = false + +@minValue(10) +@maxValue(250) +@description('The maximum number of pods per node.') +param maxPods int = 30 + +@allowed([ + 'azure' + 'kubenet' +]) +@description('The network plugin type') +param networkPlugin string = 'azure' + +@allowed([ + '' + 'Overlay' +]) +@description('The network plugin type') +param networkPluginMode string = '' + +@allowed([ + '' + 'cilium' +]) +@description('Use Cilium dataplane (requires azure networkPlugin)') +param ebpfDataplane string = '' + +@allowed([ + '' + 'azure' + 'calico' +]) +@description('The network policy to use.') +param networkPolicy string = '' + +@allowed([ + '' + 'audit' + 'deny' +]) +@description('Enable the Azure Policy addon') +param azurepolicy string = '' + +@allowed([ + 'Baseline' + 'Restricted' +]) +param azurePolicyInitiative string = 'Baseline' + +@description('The IP addresses that are allowed to access the API server') +param authorizedIPRanges array = [] + +@description('Enable private cluster') +param enablePrivateCluster bool = false + +@allowed([ + 'system' + 'none' + 'privateDnsZone' +]) +@description('Private cluster dns advertisment method, leverages the dnsApiPrivateZoneId parameter') +param privateClusterDnsMethod string = 'system' + +@description('The full Azure resource ID of the privatelink DNS zone to use for the AKS cluster API Server') +param dnsApiPrivateZoneId string = '' + +@description('The zones to use for a node pool') +param availabilityZones array = [] + +@description('Disable local K8S accounts for AAD enabled clusters') +param AksDisableLocalAccounts bool = false + +@description('Use the paid sku for SLA rather than SLO') +param AksPaidSkuForSLA bool = false + +@minLength(9) +@maxLength(18) +@description('The address range to use for pods') +param podCidr string = '10.240.100.0/22' + +@minLength(9) +@maxLength(18) +@description('The address range to use for services') +param serviceCidr string = '172.10.0.0/16' + +@minLength(7) +@maxLength(15) +@description('The IP address to reserve for DNS') +param dnsServiceIP string = '172.10.0.10' + +@minLength(9) +@maxLength(18) +@description('The address range to use for the docker bridge') +param dockerBridgeCidr string = '172.17.0.1/16' + +@description('Enable Microsoft Defender for Containers (preview)') +param defenderForContainers bool = false + +@description('Only use the system node pool') +param JustUseSystemPool bool = false + +@allowed([ + 'CostOptimised' + 'Standard' + 'HighSpec' + 'Custom' +]) +@description('The System Pool Preset sizing') +param SystemPoolType string = 'CostOptimised' + +@description('A custom system pool spec') +param SystemPoolCustomPreset object = {} + +param AutoscaleProfile object = { + 'balance-similar-node-groups': 'true' + expander: 'random' + 'max-empty-bulk-delete': '10' + 'max-graceful-termination-sec': '600' + 'max-node-provision-time': '15m' + 'max-total-unready-percentage': '45' + 'new-pod-scale-up-delay': '0s' + 'ok-total-unready-count': '3' + 'scale-down-delay-after-add': '10m' + 'scale-down-delay-after-delete': '20s' + 'scale-down-delay-after-failure': '3m' + 'scale-down-unneeded-time': '10m' + 'scale-down-unready-time': '20m' + 'scale-down-utilization-threshold': '0.5' + 'scan-interval': '10s' + 'skip-nodes-with-local-storage': 'true' + 'skip-nodes-with-system-pods': 'true' +} + +@allowed([ + 'loadBalancer' + 'managedNATGateway' + 'userAssignedNATGateway' +]) +@description('Outbound traffic type for the egress traffic of your cluster') +param aksOutboundTrafficType string = 'loadBalancer' + +@description('Create a new Nat Gateway, applies to custom networking only') +param createNatGateway bool = false + +@minValue(1) +@maxValue(16) +@description('The effective outbound IP resources of the cluster NAT gateway') +param natGwIpCount int = 2 + +@minValue(4) +@maxValue(120) +@description('Outbound flow idle timeout in minutes for NatGw') +param natGwIdleTimeout int = 30 + +@description('Configures the cluster as an OIDC issuer for use with Workload Identity') +param oidcIssuer bool = false + +@description('Installs Azure Workload Identity into the cluster') +param workloadIdentity bool = false + +@description('Assign a public IP per node for user node pools') +param enableNodePublicIP bool = false + +param warIngressNginx bool = false + +@description('System Pool presets are derived from the recommended system pool specs') +var systemPoolPresets = { + CostOptimised : { + vmSize: 'Standard_B4ms' + count: 1 + minCount: 1 + maxCount: 3 + enableAutoScaling: true + availabilityZones: [] + } + Standard : { + vmSize: 'Standard_DS2_v2' + count: 3 + minCount: 3 + maxCount: 5 + enableAutoScaling: true + availabilityZones: [ + '1' + '2' + '3' + ] + } + HighSpec : { + vmSize: 'Standard_D4s_v3' + count: 3 + minCount: 3 + maxCount: 5 + enableAutoScaling: true + availabilityZones: [ + '1' + '2' + '3' + ] + } +} + +var systemPoolBase = { + name: JustUseSystemPool ? nodePoolName : 'npsystem' + vmSize: agentVMSize + count: agentCount + mode: 'System' + osType: 'Linux' + maxPods: 30 + type: 'VirtualMachineScaleSets' + vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : json('null') + upgradeSettings: { + maxSurge: '33%' + } + nodeTaints: [ + JustUseSystemPool ? '' : 'CriticalAddonsOnly=true:NoSchedule' + ] +} + +var agentPoolProfiles = JustUseSystemPool ? array(systemPoolBase) : concat(array(union(systemPoolBase, SystemPoolType=='Custom' && SystemPoolCustomPreset != {} ? SystemPoolCustomPreset : systemPoolPresets[SystemPoolType]))) + + +output userNodePoolName string = nodePoolName +output systemNodePoolName string = JustUseSystemPool ? nodePoolName : 'npsystem' + +var akssku = AksPaidSkuForSLA ? 'Paid' : 'Free' + +var aks_addons = union({ + azurepolicy: { + config: { + version: !empty(azurepolicy) ? 'v2' : json('null') + } + enabled: !empty(azurepolicy) + } + azureKeyvaultSecretsProvider: { + config: { + enableSecretRotation: 'true' + rotationPollInterval: keyVaultAksCSIPollInterval + } + enabled: keyVaultAksCSI + } + openServiceMesh: { + enabled: openServiceMeshAddon + config: {} + } +}, createLaw && omsagent ? { + omsagent: { + enabled: createLaw && omsagent + config: { + logAnalyticsWorkspaceResourceID: createLaw && omsagent ? aks_law.id : json('null') + } + }} : {}) + +var aks_addons1 = ingressApplicationGateway ? union(aks_addons, deployAppGw ? { + ingressApplicationGateway: { + config: { + applicationGatewayId: appgw.id + } + enabled: true + } +} : { + // AKS RP will deploy the AppGateway for us (not creating custom or BYO VNET) + ingressApplicationGateway: { + enabled: true + config: { + applicationGatewayName: appgwName + subnetCIDR: '10.225.0.0/16' + } + } +}) : aks_addons + +var aks_identity = { + type: 'UserAssigned' + userAssignedIdentities: { + '${aksUai.id}': {} + } +} + +@description('Sets the private dns zone id if provided') +var aksPrivateDnsZone = privateClusterDnsMethod=='privateDnsZone' ? (!empty(dnsApiPrivateZoneId) ? dnsApiPrivateZoneId : 'system') : privateClusterDnsMethod +output aksPrivateDnsZone string = aksPrivateDnsZone + +@description('Needing to seperately declare and union this because of https://github.com/Azure/AKS-Construction/issues/344') +var managedNATGatewayProfile = { + natGatewayProfile : { + managedOutboundIPProfile: { + count: natGwIpCount + } + idleTimeoutInMinutes: natGwIdleTimeout + } +} + +@description('Needing to seperately declare and union this because of https://github.com/Azure/AKS/issues/2774') +var azureDefenderSecurityProfile = { + securityProfile : { + defender: { + logAnalyticsWorkspaceResourceId: createLaw ? aks_law.id : null + securityMonitoring: { + enabled: defenderForContainers + } + } + } +} + +var aksProperties = union({ + kubernetesVersion: kubernetesVersion + enableRBAC: true + dnsPrefix: dnsPrefix + aadProfile: enable_aad ? { + managed: true + enableAzureRBAC: enableAzureRBAC + tenantID: aad_tenant_id + } : null + apiServerAccessProfile: !empty(authorizedIPRanges) ? { + authorizedIPRanges: authorizedIPRanges + } : { + enablePrivateCluster: enablePrivateCluster + privateDNSZone: enablePrivateCluster ? aksPrivateDnsZone : '' + enablePrivateClusterPublicFQDN: enablePrivateCluster && privateClusterDnsMethod=='none' + } + agentPoolProfiles: agentPoolProfiles + workloadAutoScalerProfile: { + keda: { + enabled: kedaAddon + } + } + networkProfile: { + loadBalancerSku: 'standard' + networkPlugin: networkPlugin + #disable-next-line BCP036 //Disabling validation of this parameter to cope with empty string to indicate no Network Policy required. + networkPolicy: networkPolicy + networkPluginMode: networkPlugin=='azure' ? networkPluginMode : '' + podCidr: networkPlugin=='kubenet' || cniDynamicIpAllocation ? podCidr : json('null') + serviceCidr: serviceCidr + dnsServiceIP: dnsServiceIP + dockerBridgeCidr: dockerBridgeCidr + outboundType: aksOutboundTrafficType + ebpfDataplane: networkPlugin=='azure' ? ebpfDataplane : '' + } + disableLocalAccounts: AksDisableLocalAccounts && enable_aad + autoUpgradeProfile: {upgradeChannel: upgradeChannel} + addonProfiles: !empty(aks_addons1) ? aks_addons1 : aks_addons + autoScalerProfile: autoScale ? AutoscaleProfile : {} + oidcIssuerProfile: { + enabled: oidcIssuer + } + securityProfile: { + workloadIdentity: { + enabled: workloadIdentity + } + } + ingressProfile: { + webAppRouting: { + enabled: warIngressNginx + } + } + storageProfile: { + blobCSIDriver: { + enabled: blobCSIDriver + } + diskCSIDriver: { + enabled: diskCSIDriver + } + fileCSIDriver: { + enabled: fileCSIDriver + } + } +}, +aksOutboundTrafficType == 'managedNATGateway' ? managedNATGatewayProfile : {}, +defenderForContainers && createLaw ? azureDefenderSecurityProfile : {}, +keyVaultKmsCreateAndPrereqs || !empty(keyVaultKmsByoKeyId) ? azureKeyVaultKms : {} +) + +resource aks 'Microsoft.ContainerService/managedClusters@2022-10-02-preview' = { + name: 'aks-${resourceName}' + location: location + properties: aksProperties + identity: createAksUai ? aks_identity : { + type: 'SystemAssigned' + } + sku: { + name: 'Basic' + tier: akssku + } + dependsOn: [ + privateDnsZoneRbac + waitForKmsRbac + ] +} +output aksClusterName string = aks.name +output aksOidcIssuerUrl string = oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' + +@allowed(['Linux','Windows']) +@description('The User Node pool OS') +param osType string = 'Linux' + +@allowed(['Ubuntu','Windows2019','Windows2022']) +@description('The User Node pool OS SKU') +param osSKU string = 'Ubuntu' + +var poolName = osType == 'Linux' ? nodePoolName : take(nodePoolName, 6) + +module userNodePool 'aksagentpool.bicep' = if (!JustUseSystemPool){ + name: 'userNodePool' + params: { + AksName: aks.name + PoolName: poolName + subnetId: aksSubnetId + agentCount: agentCount + agentCountMax: agentCountMax + agentVMSize: agentVMSize + maxPods: maxPods + osDiskType: osDiskType + osType: osType + osSKU: osSKU + enableNodePublicIP: enableNodePublicIP + osDiskSizeGB: osDiskSizeGB + } +} + +@description('This output can be directly leveraged when creating a ManagedId Federated Identity') +output aksOidcFedIdentityProperties object = { + issuer: oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' + audiences: ['api://AzureADTokenExchange'] + subject: 'system:serviceaccount:ns:svcaccount' +} + +@description('The name of the managed resource group AKS uses') +output aksNodeResourceGroup string = aks.properties.nodeResourceGroup + +@description('The Azure resource id for the AKS cluster') +output aksResourceId string = aks.id + +//output aksNodePools array = [for nodepool in agentPoolProfiles: name] + +@description('Not giving Rbac at the vnet level when using private dns results in ReconcilePrivateDNS. Therefore we need to upgrade the scope when private dns is being used, because it wants to set up the dns->vnet integration.') +var uaiNetworkScopeRbac = enablePrivateCluster && !empty(dnsApiPrivateZoneId) ? 'Vnet' : 'Subnet' +module privateDnsZoneRbac './dnsZoneRbac.bicep' = if (enablePrivateCluster && !empty(dnsApiPrivateZoneId) && createAksUai) { + name: 'addPrivateK8sApiDnsContributor' + params: { + vnetId: '' + dnsZoneId: dnsApiPrivateZoneId + principalId: createAksUai ? aksUai.properties.principalId : '' + } +} + +var policySetBaseline = '/providers/Microsoft.Authorization/policySetDefinitions/a8640138-9b0a-4a28-b8cb-1666c838647d' +var policySetRestrictive = '/providers/Microsoft.Authorization/policySetDefinitions/42b8ef37-b724-4e24-bbc8-7a7708edfe00' + +resource aks_policies 'Microsoft.Authorization/policyAssignments@2020-09-01' = if (!empty(azurepolicy)) { + name: '${resourceName}-${azurePolicyInitiative}' + location: location + properties: { + policyDefinitionId: azurePolicyInitiative == 'Baseline' ? policySetBaseline : policySetRestrictive + parameters: { + excludedNamespaces: { + value: [ + 'kube-system' + 'gatekeeper-system' + 'azure-arc' + 'cluster-baseline-setting' + ] + } + effect: { + value: azurepolicy + } + } + metadata: { + assignedBy: 'Aks Construction' + } + displayName: 'Kubernetes cluster pod security ${azurePolicyInitiative} standards for Linux-based workloads' + description: 'As per: https://github.com/Azure/azure-policy/blob/master/built-in-policies/policySetDefinitions/Kubernetes/' + } +} + +@description('If automated deployment, for the 3 automated user assignments, set Principal Type on each to "ServicePrincipal" rarter than "User"') +param automatedDeployment bool = false + +@description('The principal ID to assign the AKS admin role.') +param adminPrincipalId string = '' +// for AAD Integrated Cluster wusing 'enableAzureRBAC', add Cluster admin to the current user! +var buildInAKSRBACClusterAdmin = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b') +resource aks_admin_role_assignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (enableAzureRBAC && !empty(adminPrincipalId)) { + scope: aks // Use when specifying a scope that is different than the deployment scope + name: guid(aks.id, 'aksadmin', buildInAKSRBACClusterAdmin) + properties: { + roleDefinitionId: buildInAKSRBACClusterAdmin + principalType: automatedDeployment ? 'ServicePrincipal' : 'User' + principalId: adminPrincipalId + } +} + +param fluxGitOpsAddon bool = false + +resource fluxAddon 'Microsoft.KubernetesConfiguration/extensions@2022-04-02-preview' = if(fluxGitOpsAddon) { + name: 'flux' + scope: aks + properties: { + extensionType: 'microsoft.flux' + autoUpgradeMinorVersion: true + releaseTrain: 'Stable' + scope: { + cluster: { + releaseNamespace: 'flux-system' + } + } + configurationProtectedSettings: {} + } + dependsOn: [daprExtension] //Chaining dependencies because of: https://github.com/Azure/AKS-Construction/issues/385 +} +output fluxReleaseNamespace string = fluxGitOpsAddon ? fluxAddon.properties.scope.cluster.releaseNamespace : '' + +@description('Add the Dapr extension') +param daprAddon bool = false +@description('Enable high availability (HA) mode for the Dapr control plane') +param daprAddonHA bool = false + +resource daprExtension 'Microsoft.KubernetesConfiguration/extensions@2022-04-02-preview' = if(daprAddon) { + name: 'dapr' + scope: aks + properties: { + extensionType: 'Microsoft.Dapr' + autoUpgradeMinorVersion: true + releaseTrain: 'Stable' + configurationSettings: { + 'global.ha.enabled': '${daprAddonHA}' + } + scope: { + cluster: { + releaseNamespace: 'dapr-system' + } + } + configurationProtectedSettings: {} + } +} + +output daprReleaseNamespace string = daprAddon ? daprExtension.properties.scope.cluster.releaseNamespace : '' + +/*__ ___. ______ .__ __. __ .___________. ______ .______ __ .__ __. _______ +| \/ | / __ \ | \ | | | | | | / __ \ | _ \ | | | \ | | / _____| +| \ / | | | | | | \| | | | `---| |----`| | | | | |_) | | | | \| | | | __ +| |\/| | | | | | | . ` | | | | | | | | | | / | | | . ` | | | |_ | +| | | | | `--' | | |\ | | | | | | `--' | | |\ \----.| | | |\ | | |__| | +|__| |__| \______/ |__| \__| |__| |__| \______/ | _| `._____||__| |__| \__| \______| */ + + +@description('Diagnostic categories to log') +param AksDiagCategories array = [ + 'cluster-autoscaler' + 'kube-controller-manager' + 'kube-audit-admin' + 'guard' +] + +resource AksDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (createLaw && omsagent) { + name: 'aksDiags' + scope: aks + properties: { + workspaceId: aks_law.id + logs: [for aksDiagCategory in AksDiagCategories: { + category: aksDiagCategory + enabled: true + }] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Enable Metric Alerts') +param createAksMetricAlerts bool = true + +@allowed([ + 'Short' + 'Long' +]) +@description('Which Metric polling frequency model to use') +param AksMetricAlertMetricFrequencyModel string = 'Long' + +var AlertFrequencyLookup = { + Short: { + evalFrequency: 'PT1M' + windowSize: 'PT5M' + } + Long: { + evalFrequency: 'PT15M' + windowSize: 'PT1H' + } +} +var AlertFrequency = AlertFrequencyLookup[AksMetricAlertMetricFrequencyModel] + +module aksmetricalerts './aksmetricalerts.bicep' = if (createLaw) { + name: 'aksmetricalerts' + scope: resourceGroup() + params: { + clusterName: aks.name + logAnalyticsWorkspaceName: aks_law.name + metricAlertsEnabled: createAksMetricAlerts + evalFrequency: AlertFrequency.evalFrequency + windowSize: AlertFrequency.windowSize + alertSeverity: 'Informational' + logAnalyticsWorkspaceLocation: location + } +} + +//---------------------------------------------------------------------------------- Container Insights + +@description('The Log Analytics retention period') +param retentionInDays int = 30 + +@description('The Log Analytics daily data cap (GB) (0=no limit)') +param logDataCap int = 0 + +var aks_law_name = 'log-${resourceName}' + +var createLaw = (omsagent || deployAppGw || azureFirewalls || CreateNetworkSecurityGroups || defenderForContainers) + +resource aks_law 'Microsoft.OperationalInsights/workspaces@2022-10-01' = if (createLaw) { + name: aks_law_name + location: location + properties : union({ + retentionInDays: retentionInDays + }, + logDataCap>0 ? { workspaceCapping: { + dailyQuotaGb: logDataCap + }} : {} + ) +} + + +resource containerLogsV2_Basiclogs 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = if(containerLogsV2BasicLogs){ + name: '${aks_law_name}/ContainerLogV2' + properties: { + plan: 'Basic' + } + dependsOn: [ + aks + ] +} + +//This role assignment enables AKS->LA Fast Alerting experience +var MonitoringMetricsPublisherRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') +resource FastAlertingRole_Aks_Law 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (omsagent) { + scope: aks + name: guid(aks.id, 'omsagent', MonitoringMetricsPublisherRole) + properties: { + roleDefinitionId: MonitoringMetricsPublisherRole + principalId: aks.properties.addonProfiles.omsagent.identity.objectId + principalType: 'ServicePrincipal' + } +} + +output LogAnalyticsName string = (createLaw) ? aks_law.name : '' +output LogAnalyticsGuid string = (createLaw) ? aks_law.properties.customerId : '' +output LogAnalyticsId string = (createLaw) ? aks_law.id : '' + +//---------------------------------------------------------------------------------- AKS events with Event Grid +// Supported events : https://docs.microsoft.com/en-gb/azure/event-grid/event-schema-aks?tabs=event-grid-event-schema#available-event-types + +@description('Create an Event Grid System Topic for AKS events') +param createEventGrid bool = false + +resource eventGrid 'Microsoft.EventGrid/systemTopics@2021-12-01' = if(createEventGrid) { + name: 'evgt-${aks.name}' + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + source: aks.id + topicType: 'Microsoft.ContainerService.ManagedClusters' + } +} + +output eventGridName string = createEventGrid ? eventGrid.name : '' + +resource eventGridDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (createLaw && createEventGrid) { + name: 'eventGridDiags' + scope: eventGrid + properties: { + workspaceId:aks_law.id + logs: [ + { + category: 'DeliveryFailures' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Enable usage and telemetry feedback to Microsoft.') +param enableTelemetry bool = true + +var telemetryId = '3c1e2fc6-1c4b-44f9-8694-25d00ae30a3a-${location}' + +/*.___________. _______ __ _______ .___ ___. _______ .___________..______ ____ ____ _______ _______ .______ __ ______ ____ ____ .___ ___. _______ .__ __. .___________. +| || ____|| | | ____|| \/ | | ____|| || _ \ \ \ / / | \ | ____|| _ \ | | / __ \ \ \ / / | \/ | | ____|| \ | | | | +`---| |----`| |__ | | | |__ | \ / | | |__ `---| |----`| |_) | \ \/ / | .--. || |__ | |_) | | | | | | | \ \/ / | \ / | | |__ | \| | `---| |----` + | | | __| | | | __| | |\/| | | __| | | | / \_ _/ | | | || __| | ___/ | | | | | | \_ _/ | |\/| | | __| | . ` | | | + | | | |____ | `----.| |____ | | | | | |____ | | | |\ \----. | | | '--' || |____ | | | `----.| `--' | | | | | | | | |____ | |\ | | | + |__| |_______||_______||_______||__| |__| |_______| |__| | _| `._____| |__| |_______/ |_______|| _| |_______| \______/ |__| |__| |__| |_______||__| \__| |__| */ + +// Telemetry Deployment +resource telemetrydeployment 'Microsoft.Resources/deployments@2021-04-01' = if (enableTelemetry) { + name: telemetryId + properties: { + mode: 'Incremental' + template: { + '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' + contentVersion: '1.0.0.0' + resources: {} + } + } +} + +//ACSCII Art link : https://textkool.com/en/ascii-art-generator?hl=default&vl=default&font=Star%20Wars&text=changeme diff --git a/templates/common/infra/bicep/core/host/aks/network.bicep b/templates/common/infra/bicep/core/host/aks/network.bicep new file mode 100644 index 00000000000..0ec70091a08 --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/network.bicep @@ -0,0 +1,506 @@ +param resourceName string +param location string = resourceGroup().location + +param networkPluginIsKubenet bool = false +param aksPrincipleId string = '' + +param vnetAddressPrefix string +param vnetAksSubnetAddressPrefix string + +//Nsg +param workspaceName string = '' +param workspaceResourceGroupName string = '' +param networkSecurityGroups bool = true + +//Firewall +param azureFirewalls bool = false +param azureFirewallsSku string = 'Basic' +param azureFirewallsManagementSeperation bool = azureFirewalls && azureFirewallsSku=='Basic' +param vnetFirewallSubnetAddressPrefix string = '' +param vnetFirewallManagementSubnetAddressPrefix string = '' + +//Ingress +param ingressApplicationGateway bool = false +param ingressApplicationGatewayPublic bool = false +param vnetAppGatewaySubnetAddressPrefix string ='' + +//Private Link +param privateLinks bool = false +param privateLinkSubnetAddressPrefix string = '' +param privateLinkAcrId string = '' +param privateLinkAkvId string = '' + +//ACR +param acrPrivatePool bool = false +param acrAgentPoolSubnetAddressPrefix string = '' + +//NatGatewayEgress +param natGateway bool = false +param natGatewayPublicIps int = 2 +param natGatewayIdleTimeoutMins int = 30 + +//Bastion +param bastion bool =false +param bastionSubnetAddressPrefix string = '' + +@description('Used by the Bastion Public IP') +param availabilityZones array = [] + + +var bastion_subnet_name = 'AzureBastionSubnet' +var bastion_baseSubnet = { + name: bastion_subnet_name + properties: { + addressPrefix: bastionSubnetAddressPrefix + } +} +var bastion_subnet = bastion && networkSecurityGroups ? union(bastion_baseSubnet, nsgBastion.outputs.nsgSubnetObj) : bastion_baseSubnet + +var acrpool_subnet_name = 'acrpool-sn' +var acrpool_baseSubnet = { + name: acrpool_subnet_name + properties: { + addressPrefix: acrAgentPoolSubnetAddressPrefix + } +} +var acrpool_subnet = privateLinks && networkSecurityGroups ? union(acrpool_baseSubnet, nsgAcrPool.outputs.nsgSubnetObj) : acrpool_baseSubnet + +var private_link_subnet_name = 'privatelinks-sn' +var private_link_baseSubnet = { + name: private_link_subnet_name + properties: { + addressPrefix: privateLinkSubnetAddressPrefix + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } +} +var private_link_subnet = privateLinks && networkSecurityGroups ? union(private_link_baseSubnet, nsgPrivateLinks.outputs.nsgSubnetObj) : private_link_baseSubnet + + +var appgw_subnet_name = 'appgw-sn' +var appgw_baseSubnet = { + name: appgw_subnet_name + properties: { + addressPrefix: vnetAppGatewaySubnetAddressPrefix + } +} +var appgw_subnet = ingressApplicationGateway && networkSecurityGroups ? union(appgw_baseSubnet, nsgAppGw.outputs.nsgSubnetObj) : appgw_baseSubnet + +var fw_subnet_name = 'AzureFirewallSubnet' // Required by FW +var fw_subnet = { + name: fw_subnet_name + properties: { + addressPrefix: vnetFirewallSubnetAddressPrefix + } +} + +/// ---- Firewall VNET config +module calcAzFwIp './calcAzFwIp.bicep' = if (azureFirewalls) { + name: 'calcAzFwIp' + params: { + vnetFirewallSubnetAddressPrefix: vnetFirewallSubnetAddressPrefix + } +} + +var fwmgmt_subnet_name = 'AzureFirewallManagementSubnet' // Required by FW +var fwmgmt_subnet = { + name: fwmgmt_subnet_name + properties: { + addressPrefix: vnetFirewallManagementSubnetAddressPrefix + } +} + +var routeFwTableName = 'rt-afw-${resourceName}' +resource vnet_udr 'Microsoft.Network/routeTables@2022-07-01' = if (azureFirewalls) { + name: routeFwTableName + location: location + properties: { + routes: [ + { + name: 'AKSNodesEgress' + properties: { + addressPrefix: '0.0.0.0/1' + nextHopType: 'VirtualAppliance' + nextHopIpAddress: azureFirewalls ? calcAzFwIp.outputs.FirewallPrivateIp : null + } + } + ] + } +} + +var contributorRoleId = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') + +@description('Required for kubenet networking.') +resource vnet_udr_rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (azureFirewalls && !empty(aksPrincipleId) && networkPluginIsKubenet) { + scope: vnet_udr + name: guid(vnet_udr.id, aksPrincipleId, contributorRoleId) + properties: { + principalId: aksPrincipleId + roleDefinitionId: contributorRoleId + principalType: 'ServicePrincipal' + } +} + +var aks_subnet_name = 'aks-sn' +var aks_baseSubnet = { + name: aks_subnet_name + properties: union({ + addressPrefix: vnetAksSubnetAddressPrefix + }, privateLinks ? { + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } : {}, natGateway ? { + natGateway: { + id: natGw.id + } + } : {}, azureFirewalls ? { + routeTable: { + id: vnet_udr.id //resourceId('Microsoft.Network/routeTables', routeFwTableName) + } + }: {}) +} + +var aks_subnet = networkSecurityGroups ? union(aks_baseSubnet, nsgAks.outputs.nsgSubnetObj) : aks_baseSubnet + +var subnets = union( + array(aks_subnet), + azureFirewalls ? array(fw_subnet) : [], + privateLinks ? array(private_link_subnet) : [], + acrPrivatePool ? array(acrpool_subnet) : [], + bastion ? array(bastion_subnet) : [], + ingressApplicationGateway ? array(appgw_subnet) : [], + azureFirewallsManagementSeperation ? array(fwmgmt_subnet) : [] +) +output debugSubnets array = subnets + +var vnetName = 'vnet-${resourceName}' +resource vnet 'Microsoft.Network/virtualNetworks@2021-02-01' = { + name: vnetName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + vnetAddressPrefix + ] + } + subnets: subnets + } +} +output vnetId string = vnet.id +output vnetName string = vnet.name +output aksSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, aks_subnet_name) +output fwSubnetId string = azureFirewalls ? '${vnet.id}/subnets/${fw_subnet_name}' : '' +output fwMgmtSubnetId string = azureFirewalls ? '${vnet.id}/subnets/${fwmgmt_subnet_name}' : '' +output acrPoolSubnetId string = acrPrivatePool ? '${vnet.id}/subnets/${acrpool_subnet_name}' : '' +output appGwSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, appgw_subnet_name) +output privateLinkSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, private_link_subnet_name) + +module aks_vnet_con 'networksubnetrbac.bicep' = if (!empty(aksPrincipleId)) { + name: '${resourceName}-subnetRbac' + params: { + servicePrincipalId: aksPrincipleId + subnetName: aks_subnet_name + vnetName: vnet.name + } +} + +/* -------------------------------------------------------------------------- Private Link for ACR */ +var privateLinkAcrName = 'pl-acr-${resourceName}' +resource privateLinkAcr 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAcrId)) { + name: privateLinkAcrName + location: location + properties: { + customNetworkInterfaceName: 'nic-${privateLinkAcrName}' + privateLinkServiceConnections: [ + { + name: 'Acr-Connection' + properties: { + privateLinkServiceId: privateLinkAcrId + groupIds: [ + 'registry' + ] + } + } + ] + subnet: { + id: '${vnet.id}/subnets/${private_link_subnet_name}' + } + } +} + +resource privateDnsAcr 'Microsoft.Network/privateDnsZones@2020-06-01' = if (!empty(privateLinkAcrId)) { + name: 'privatelink.azurecr.io' + location: 'global' +} + +var privateDnsAcrLinkName = 'vnet-dnscr-${resourceName}' +resource privateDnsAcrLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (!empty(privateLinkAcrId)) { + parent: privateDnsAcr + name: privateDnsAcrLinkName + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnet.id + } + } +} + +resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAcrId)) { + parent: privateLinkAcr + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'vnet-pl-acr' + properties: { + privateDnsZoneId: privateDnsAcr.id + } + } + ] + } +} + + +/* -------------------------------------------------------------------------- Private Link for KeyVault */ +var privateLinkAkvName = 'pl-akv-${resourceName}' +resource privateLinkAkv 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAkvId)) { + name: privateLinkAkvName + location: location + properties: { + customNetworkInterfaceName: 'nic-${privateLinkAkvName}' + privateLinkServiceConnections: [ + { + name: 'Akv-Connection' + properties: { + privateLinkServiceId: privateLinkAkvId + groupIds: [ + 'vault' + ] + } + } + ] + subnet: { + id: '${vnet.id}/subnets/${private_link_subnet_name}' + } + } +} + +resource privateDnsAkv 'Microsoft.Network/privateDnsZones@2020-06-01' = if (!empty(privateLinkAkvId)) { + name: 'privatelink.vaultcore.azure.net' + location: 'global' +} + +var privateDnsAkvLinkName = 'vnet-dnscr-${resourceName}' +resource privateDnsAkvLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (!empty(privateLinkAkvId)) { + parent: privateDnsAkv + name: privateDnsAkvLinkName + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnet.id + } + } +} + +resource privateDnsAkvZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAkvId)) { + parent: privateLinkAkv + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'vnet-pl-akv' + properties: { + privateDnsZoneId: privateDnsAkv.id + } + } + ] + } +} + +param bastionHostName string = 'bas-${resourceName}' +var publicIpAddressName = 'pip-${bastionHostName}' + +@allowed([ + 'Standard' + 'Basic' +]) +param bastionSku string = 'Standard' + +resource bastionPip 'Microsoft.Network/publicIPAddresses@2021-03-01' = if(bastion) { + name: publicIpAddressName + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIPAllocationMethod: 'Static' + } +} + +resource bastionHost 'Microsoft.Network/bastionHosts@2021-05-01' = if(bastion) { + name: bastionHostName + location: location + sku: { + name: bastionSku + } + properties: { + enableTunneling: true + ipConfigurations: [ + { + name: 'IpConf' + properties: { + subnet: { + id: '${vnet.id}/subnets/${bastion_subnet_name}' + } + publicIPAddress: { + id: bastionPip.id + } + } + } + ] + } +} + +resource log 'Microsoft.OperationalInsights/workspaces@2021-06-01' existing = if(networkSecurityGroups && !empty(workspaceName)) { + name: workspaceName + scope: resourceGroup(workspaceResourceGroupName) +} + +param CreateNsgFlowLogs bool = false + +var flowLogStorageRawName = replace(toLower('stflow${resourceName}${uniqueString(resourceGroup().id, resourceName)}'),'-','') +var flowLogStorageName = length(flowLogStorageRawName) > 24 ? substring(flowLogStorageRawName, 0, 24) : flowLogStorageRawName +resource flowLogStor 'Microsoft.Storage/storageAccounts@2021-08-01' = if(CreateNsgFlowLogs && networkSecurityGroups) { + name: flowLogStorageName + kind: 'StorageV2' + sku: { + name: 'Standard_LRS' + } + location: location + properties: { + minimumTlsVersion: 'TLS1_2' + } +} + +//NSG's +module nsgAks 'nsg.bicep' = if(networkSecurityGroups) { + name: 'nsgAks' + params: { + location: location + resourceName: '${aks_subnet_name}-${resourceName}' + workspaceId: !empty(workspaceName) ? log.properties.customerId : '' + workspaceRegion: !empty(workspaceName) ? log.location : '' + workspaceResourceId: !empty(workspaceName) ? log.id : '' + ruleInAllowInternetHttp: true + ruleInAllowInternetHttps: true + ruleInDenySsh: true + FlowLogStorageAccountId: CreateNsgFlowLogs ? flowLogStor.id : '' + } +} + +module nsgAcrPool 'nsg.bicep' = if(acrPrivatePool && networkSecurityGroups) { + name: 'nsgAcrPool' + params: { + location: location + resourceName: '${acrpool_subnet_name}-${resourceName}' + workspaceId: !empty(workspaceName) ? log.properties.customerId : '' + workspaceRegion: !empty(workspaceName) ? log.location : '' + workspaceResourceId: !empty(workspaceName) ? log.id : '' + FlowLogStorageAccountId: CreateNsgFlowLogs ? flowLogStor.id : '' + } + dependsOn: [ + nsgAks + ] +} + +module nsgAppGw 'nsg.bicep' = if(ingressApplicationGateway && networkSecurityGroups) { + name: 'nsgAppGw' + params: { + location: location + resourceName: '${appgw_subnet_name}-${resourceName}' + workspaceId: !empty(workspaceName) ? log.properties.customerId : '' + workspaceRegion: !empty(workspaceName) ? log.location : '' + workspaceResourceId: !empty(workspaceName) ? log.id : '' + ruleInAllowInternetHttp: ingressApplicationGatewayPublic + ruleInAllowInternetHttps: ingressApplicationGatewayPublic + ruleInAllowGwManagement: true + ruleInAllowAzureLoadBalancer: true + ruleInDenyInternet: true + ruleInGwManagementPort: '65200-65535' + FlowLogStorageAccountId: CreateNsgFlowLogs ? flowLogStor.id : '' + } + dependsOn: [ + nsgAcrPool + ] +} + +module nsgBastion 'nsg.bicep' = if(bastion && networkSecurityGroups) { + name: 'nsgBastion' + params: { + location: location + resourceName: '${bastion_subnet_name}-${resourceName}' + workspaceId: !empty(workspaceName) ? log.properties.customerId : '' + workspaceRegion: !empty(workspaceName) ? log.location : '' + workspaceResourceId: !empty(workspaceName) ? log.id : '' + ruleInAllowBastionHostComms: true + ruleInAllowInternetHttps: true + ruleInAllowGwManagement: true + ruleInAllowAzureLoadBalancer: true + ruleOutAllowBastionComms: true + ruleInGwManagementPort: '443' + FlowLogStorageAccountId: CreateNsgFlowLogs ? flowLogStor.id : '' + } + dependsOn: [ + nsgAppGw + ] +} + +module nsgPrivateLinks 'nsg.bicep' = if(privateLinks && networkSecurityGroups) { + name: 'nsgPrivateLinks' + params: { + location: location + resourceName: '${private_link_subnet_name}-${resourceName}' + workspaceId: !empty(workspaceName) ? log.properties.customerId : '' + workspaceRegion: !empty(workspaceName) ? log.location : '' + workspaceResourceId: !empty(workspaceName) ? log.id : '' + FlowLogStorageAccountId: CreateNsgFlowLogs ? flowLogStor.id : '' + } + dependsOn: [ + nsgBastion + ] +} + +resource natGwIp 'Microsoft.Network/publicIPAddresses@2021-08-01' = [for i in range(0, natGatewayPublicIps): if(natGateway) { + name: 'pip-${natGwName}-${i+1}' + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIPAllocationMethod: 'Static' + } +}] + +var natGwName = 'ng-${resourceName}' +resource natGw 'Microsoft.Network/natGateways@2021-08-01' = if(natGateway) { + name: natGwName + location: location + sku: { + name: 'Standard' + } + zones: !empty(availabilityZones) ? availabilityZones : [] + properties: { + publicIpAddresses: [for i in range(0, natGatewayPublicIps): { + id: natGwIp[i].id + }] + idleTimeoutInMinutes: natGatewayIdleTimeoutMins + } + dependsOn: [ + natGwIp + ] +} + diff --git a/templates/common/infra/bicep/core/host/aks/networksubnetrbac.bicep b/templates/common/infra/bicep/core/host/aks/networksubnetrbac.bicep new file mode 100644 index 00000000000..b19cfef527a --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/networksubnetrbac.bicep @@ -0,0 +1,19 @@ +param vnetName string +param subnetName string +param servicePrincipalId string + +var networkContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') + +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = { + name: '${vnetName}/${subnetName}' +} + +resource aks_vnet_cont 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subnet.id, servicePrincipalId, networkContributorRole) + scope: subnet + properties: { + roleDefinitionId: networkContributorRole + principalId: servicePrincipalId + principalType: 'ServicePrincipal' + } +} diff --git a/templates/common/infra/bicep/core/host/aks/networkwatcherflowlog.bicep b/templates/common/infra/bicep/core/host/aks/networkwatcherflowlog.bicep new file mode 100644 index 00000000000..8c77e80d9dd --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/networkwatcherflowlog.bicep @@ -0,0 +1,46 @@ +param name string +param location string = resourceGroup().location +param nsgId string +param storageId string +param trafficAnalytics bool +param trafficAnalyticsInterval int = 60 + +@description('The resource guid of the attached workspace.') +param workspaceId string = '' + +@description('Resource Id of the attached workspace.') +param workspaceResourceId string = '' +param workspaceRegion string = resourceGroup().location + +resource networkWatcher 'Microsoft.Network/networkWatchers@2022-01-01' = { + name: 'NetworkWatcher_${location}' + location: location + properties: {} +} + +resource nsgFlowLogs 'Microsoft.Network/networkWatchers/flowLogs@2021-05-01' = { + name: '${networkWatcher.name}/${name}' + location: location + properties: { + targetResourceId: nsgId + storageId: storageId + enabled: true + retentionPolicy: { + days: 2 + enabled: true + } + format: { + type: 'JSON' + version: 2 + } + flowAnalyticsConfiguration: { + networkWatcherFlowAnalyticsConfiguration: { + enabled: trafficAnalytics + workspaceId: workspaceId + trafficAnalyticsInterval: trafficAnalyticsInterval + workspaceRegion: workspaceRegion + workspaceResourceId: workspaceResourceId + } + } + } +} diff --git a/templates/common/infra/bicep/core/host/aks/nsg.bicep b/templates/common/infra/bicep/core/host/aks/nsg.bicep new file mode 100644 index 00000000000..e90e57f5d2d --- /dev/null +++ b/templates/common/infra/bicep/core/host/aks/nsg.bicep @@ -0,0 +1,283 @@ +param resourceName string +param location string = resourceGroup().location +param workspaceId string = '' +param workspaceResourceId string = '' +param workspaceRegion string = resourceGroup().location + +var nsgName = 'nsg-${resourceName}' + +resource nsg 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: nsgName + location: location +} +output nsgId string = nsg.id + +param ruleInAllowGwManagement bool = false +param ruleInGwManagementPort string = '443,65200-65535' +resource ruleAppGwManagement 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInAllowGwManagement) { + parent: nsg + name: 'Allow_AppGatewayManagement' + properties: { + protocol: '*' + sourcePortRange: '*' + destinationPortRange: ruleInGwManagementPort + sourceAddressPrefix: 'GatewayManager' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } +} + +param ruleInAllowAzureLoadBalancer bool = false +resource ruleAzureLoadBalancer 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if (ruleInAllowAzureLoadBalancer) { + parent: nsg + name: 'Allow_AzureLoadBalancer' + properties: { + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleInDenyInternet bool = false +resource ruleDenyInternet 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInDenyInternet) { + parent: nsg + name: 'Deny_AllInboundInternet' + properties: { + description: 'Azure infrastructure communication' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 4096 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleInAllowInternetHttp bool = false +resource ruleInternetHttp 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInAllowInternetHttp) { + parent: nsg + name: 'Allow_Internet_Http' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 200 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '80' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleInAllowInternetHttps bool = false +resource ruleInternetHttps 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInAllowInternetHttps) { + parent: nsg + name: 'Allow_Internet_Https' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 210 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '443' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleInAllowBastionHostComms bool = false +resource ruleBastionHost 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInAllowBastionHostComms) { + parent: nsg + name: 'Allow_Bastion_Host_Communication' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 700 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '8080' + '5701' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleOutAllowBastionComms bool = false +resource ruleBastionEgressSshRdp 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleOutAllowBastionComms) { + parent: nsg + name: 'Allow_SshRdp_Outbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 200 + direction: 'Outbound' + sourcePortRanges: [] + destinationPortRanges: [ + '22' + '3389' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +resource ruleBastionEgressAzure 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleOutAllowBastionComms) { + parent: nsg + name: 'Allow_Azure_Cloud_Outbound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 210 + direction: 'Outbound' + sourcePortRanges: [] + destinationPortRanges: [ + '443' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +resource ruleBastionEgressBastionComms 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleOutAllowBastionComms) { + parent: nsg + name: 'Allow_Bastion_Communication' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 220 + direction: 'Outbound' + sourcePortRanges: [] + destinationPortRanges: [ + '8080' + '5701' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +resource ruleBastionEgressSessionInfo 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleOutAllowBastionComms) { + parent: nsg + name: 'Allow_Get_Session_Info' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 230 + direction: 'Outbound' + sourcePortRanges: [] + destinationPortRanges: [ + '80' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param ruleInDenySsh bool = false +resource ruleSshIngressDeny 'Microsoft.Network/networkSecurityGroups/securityRules@2020-11-01' = if(ruleInDenySsh) { + parent: nsg + name: 'DenySshInbound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 100 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '22' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } +} + +param NsgDiagnosticCategories array = [ + 'NetworkSecurityGroupEvent' + 'NetworkSecurityGroupRuleCounter' +] + +resource nsgDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceResourceId)) { + name: 'diags-${nsgName}' + scope: nsg + properties: { + workspaceId: workspaceResourceId + logs: [for diagCategory in NsgDiagnosticCategories: { + category: diagCategory + enabled: true + }] + } +} + +//If multiple NSG's are trying to add flow logs at the same, this will result in CONFLICT: AnotherOperationInProgress +//Therefore advised to create FlowLogs outside of NSG creation, to better coordinate the creation - or sequence the NSG creation with DependsOn +param FlowLogStorageAccountId string = '' +param FlowLogTrafficAnalytics bool = !empty(FlowLogStorageAccountId) +module nsgFlow 'networkwatcherflowlog.bicep' = if(!empty(FlowLogStorageAccountId)) { + name: 'flow-${nsgName}' + scope: resourceGroup('NetworkWatcherRG') + params: { + location:location + name: 'flowNsg-${nsgName}' + nsgId: nsg.id + storageId: FlowLogStorageAccountId + trafficAnalytics: FlowLogTrafficAnalytics + workspaceId: workspaceId + workspaceResourceId: workspaceResourceId + workspaceRegion: workspaceRegion + } +} + +output nsgSubnetObj object = { + properties: { + networkSecurityGroup: { + id: nsg.id + } + } +} diff --git a/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/azure.yaml b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/azure.yaml new file mode 100644 index 00000000000..16167b806a1 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/azure.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: todo-nodejs-mongo-aks +metadata: + template: todo-nodejs-mongo-aks@0.0.1-beta +services: + web: + project: ../../web/react-fluent + dist: build + language: js + host: aks + api: + project: ../../api/js + language: js + host: aks diff --git a/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.bicep b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.bicep new file mode 100644 index 00000000000..b4a1f752af2 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.bicep @@ -0,0 +1,102 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the the environment which is used to generate a short unique hash used in all resources.') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +// Optional parameters to override the default azd resource naming conventions. Update the main.parameters.json file to provide values. e.g.,: +// "resourceGroupName": { +// "value": "myGroupName" +// } +param applicationInsightsDashboardName string = '' +param applicationInsightsName string = '' +param cosmosAccountName string = '' +param cosmosDatabaseName string = '' +param keyVaultName string = '' +param logAnalyticsName string = '' +param resourceGroupName string = '' + +@description('Id of the user or app to assign application roles') +param principalId string = '' + +var abbrs = loadJsonContent('../../../../../../common/infra/bicep/abbreviations.json') +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) +var tags = { 'azd-env-name': environmentName } + +// Organize resources in a resource group +resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: !empty(resourceGroupName) ? resourceGroupName : '${abbrs.resourcesResourceGroups}${environmentName}' + location: location + tags: tags +} + +module cluster '../../../../../../common/infra/bicep/core/host/aks/main.bicep' = { + name: 'aks' + scope: rg + params: { + location: location + resourceName: resourceToken + upgradeChannel: 'stable' + warIngressNginx: true + adminPrincipalId: principalId + acrPushRolePrincipalId: principalId + } +} + +// The application database +module cosmos '../../../../../common/infra/bicep/app/cosmos-mongo-db.bicep' = { + name: 'cosmos' + scope: rg + params: { + accountName: !empty(cosmosAccountName) ? cosmosAccountName : '${abbrs.documentDBDatabaseAccounts}${resourceToken}' + databaseName: cosmosDatabaseName + location: location + tags: tags + keyVaultName: keyVault.outputs.name + } +} + +// Store secrets in a keyvault +module keyVault '../../../../../../common/infra/bicep/core/security/keyvault.bicep' = { + name: 'keyvault' + scope: rg + params: { + name: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${resourceToken}' + location: location + tags: tags + principalId: principalId + } +} + +// Monitor application with Azure Monitor +module monitoring '../../../../../../common/infra/bicep/core/monitor/monitoring.bicep' = { + name: 'monitoring' + scope: rg + params: { + location: location + tags: tags + logAnalyticsName: !empty(logAnalyticsName) ? logAnalyticsName : '${abbrs.operationalInsightsWorkspaces}${resourceToken}' + applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${resourceToken}' + applicationInsightsDashboardName: !empty(applicationInsightsDashboardName) ? applicationInsightsDashboardName : '${abbrs.portalDashboards}${resourceToken}' + } +} + +// Data outputs +output AZURE_COSMOS_CONNECTION_STRING_KEY string = cosmos.outputs.connectionStringKey +output AZURE_COSMOS_DATABASE_NAME string = cosmos.outputs.databaseName + +// App outputs +output APPLICATIONINSIGHTS_CONNECTION_STRING string = monitoring.outputs.applicationInsightsConnectionString +output AZURE_KEY_VAULT_ENDPOINT string = keyVault.outputs.endpoint +output AZURE_KEY_VAULT_NAME string = keyVault.outputs.name +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output REACT_APP_API_BASE_URL string = '' +output REACT_APP_APPLICATIONINSIGHTS_CONNECTION_STRING string = monitoring.outputs.applicationInsightsConnectionString +output REACT_APP_WEB_BASE_URL string = '' +output SERVICE_API_ENDPOINTS array = [] diff --git a/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.parameters.json b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.parameters.json new file mode 100644 index 00000000000..67ad8524c44 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/infra/main.parameters.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + }, + "principalId": { + "value": "${AZURE_PRINCIPAL_ID}" + } + } +} \ No newline at end of file diff --git a/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/repo.yaml b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/repo.yaml new file mode 100644 index 00000000000..558c13f2504 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.repo/bicep/repo.yaml @@ -0,0 +1,202 @@ +templateApi: 1.0.0 +metadata: + type: repo + name: todo-nodejs-mongo-aks + description: ToDo Application with a Node.js API and Azure Cosmos DB API for MongoDB hosted in AKS + +repo: + includeProjectAssets: false + + remotes: + - name: wbreza-main + url: git@github.com:wbreza/todo-nodejs-mongo-aks.git + - name: wbreza-staging + url: git@github.com:wbreza/todo-nodejs-mongo-aks.git + branch: staging + + rewrite: + rules: + - from: ../../../../../../common/infra/bicep/core + to: ./core + patterns: + - "**/*.bicep" + + - from: ../../../../../common/infra/bicep/app + to: ./app + patterns: + - "**/*.bicep" + + - from: ../../../../../common/infra/bicep/core + to: ../core + patterns: + - "**/*.bicep" + + - from: ../../../../../common/infra/shared/gateway/apim + to: ./ + patterns: + - apim-api.bicep + + # app service modules + - from: ../../../../../../common/infra/bicep + to: ../ + patterns: + - "**/*.bicep" + ignore: + - "**/main.bicep" + + # main.bicep + - from: ../../../../../../common/infra/bicep + to: ./ + patterns: + - "**/main.bicep" + + - from: ../../api/js + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" + + - from: web-appservice.bicep + to: web.bicep + patterns: + - "**/main.bicep" + + - from: api-appservice-node.bicep + to: api.bicep + patterns: + - "**/main.bicep" + + - from: cosmos-mongo-db.bicep + to: db.bicep + patterns: + - "**/main.bicep" + + - from: "PLACEHOLDERIACTOOLS" + to: "" + patterns: + - "README.md" + + - from: "PLACEHOLDER_TITLE" + to: "ToDo Application with a Node.js API and Azure Cosmos DB API for MongoDB on Azure App Service" + patterns: + - "README.md" + + - from: "PLACEHOLDER_DESCRIPTION" + to: "using Bicep as the IaC provider" + patterns: + - "README.md" + + - from: ../../../../api/common/openapi.yaml + to: ../../src/api/openapi.yaml + patterns: + - "apim-api.bicep" + + assets: + # Common assets + + # Infra + - from: ./infra/ + to: ./infra + + - from: ../../../../common/infra/bicep/app/web-appservice.bicep + to: ./infra/app/web.bicep + + - from: ../../../../common/infra/bicep/app/api-appservice-node.bicep + to: ./infra/app/api.bicep + + - from: ../../../../common/infra/bicep/app/apim-api.bicep + to: ./infra/app/apim-api.bicep + + - from: ../../../../../common/infra/shared/gateway/apim/apim-api-policy.xml + to: ./infra/app/apim-api-policy.xml + + - from: ../../../../common/infra/bicep/app/cosmos-mongo-db.bicep + to: ./infra/app/db.bicep + + - from: ./../../ + to: ./ + ignore: + - ".repo/**/*" + - "repo.y[a]ml" + - "azure.y[a]ml" + + # openapi.yaml to root + - from: ../../../../api/common + to: ./ + patterns: + - openapi.yaml + + # openapi.yaml to api root + - from: ../../../../api/common + to: ./src/api + patterns: + - openapi.yaml + + # Templates common + - from: ../../../../../common + to: ./ + ignore: + - .github/**/* + - .devcontainer/**/* + - "infra/**/*" + - .azdo/pipelines/*/azure-dev.yml + + # AzDo workflows for bicep + - from: ../../../../../common/.azdo/pipelines/bicep/azure-dev.yml + to: ./.azdo/pipelines/azure-dev.yml + + # Github workflows for bicep + - from: ../../../../../common/.github/workflows/bicep + to: ./.github/workflows + + # azd core modules + - from: ../../../../../common/infra/bicep + to: ./infra + + # .devcontainer common (devcontainer.json) + - from: ../../../../../common/.devcontainer/devcontainer.json/nodejs/devcontainer.json + to: ./.devcontainer/devcontainer.json + + # .devcontainer common (Dockerfile) + - from: ../../../../../common/.devcontainer/Dockerfile/base + to: ./.devcontainer + + # Assets common + - from: ../../../../common/assets + to: ./assets + + # Tests common + - from: ../../../../common/tests + to: ./tests + + # Auth JS common + - from: ../../../../common/auth/js + to: ./src/api/src + + # Node JS API + - from: ../../../../api/js + to: ./src/api + ignore: + - "dist/**/*" + - "coverage/**/*" + - "node_modules/**/*" + - "**/*.log" + + # React Frontend + - from: ../../../../web/react-fluent + to: ./src/web + ignore: + - "build/**/*" + - "node_modules/**/*" + + # Infra + - from: ./infra/ + to: ./infra + + # Azure.yml + - from: ./azure.yaml + to: ./azure.yaml diff --git a/templates/todo/projects/nodejs-mongo-aks/.vscode/launch.json b/templates/todo/projects/nodejs-mongo-aks/.vscode/launch.json new file mode 100644 index 00000000000..709bf3d1c0f --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.vscode/launch.json @@ -0,0 +1,47 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug Web", + "request": "launch", + "type": "msedge", + "webRoot": "${workspaceFolder}/src/web/src", + "url": "http://localhost:3000", + "sourceMapPathOverrides": { + "webpack:///src/*": "${webRoot}/*" + }, + }, + + { + "name": "Debug API", + "request": "launch", + "runtimeArgs": [ + "run", + "start" + ], + "runtimeExecutable": "npm", + "skipFiles": [ + "/**" + ], + "type": "pwa-node", + "cwd": "${workspaceFolder}/src/api", + "envFile": "${input:dotEnvFilePath}", + "env": { + "NODE_ENV": "development" + }, + "preLaunchTask": "Restore API", + "outputCapture": "std" + }, + ], + + "inputs": [ + { + "id": "dotEnvFilePath", + "type": "command", + "command": "azure-dev.commands.getDotEnvFilePath" + } + ] +} diff --git a/templates/todo/projects/nodejs-mongo-aks/.vscode/tasks.json b/templates/todo/projects/nodejs-mongo-aks/.vscode/tasks.json new file mode 100644 index 00000000000..935126d74d7 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/.vscode/tasks.json @@ -0,0 +1,92 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Start Web", + "type": "dotenv", + "targetTasks": [ + "Restore Web", + "Web npm start" + ], + "file": "${input:dotEnvFilePath}" + }, + { + "label": "Restore Web", + "type": "shell", + "command": "azd restore --service web", + "presentation": { + "reveal": "silent" + }, + "problemMatcher": [] + }, + { + "label": "Web npm start", + "detail": "Helper task--use 'Start Web' task to ensure environment is set up correctly", + "type": "shell", + "command": "npm run start", + "options": { + "cwd": "${workspaceFolder}/src/web/", + "env": { + "REACT_APP_API_BASE_URL": "http://localhost:3100", + "BROWSER": "none" + } + }, + "presentation": { + "panel": "dedicated", + }, + "problemMatcher": [] + }, + + { + "label": "Start API", + "type": "dotenv", + "targetTasks": [ + "Restore API", + "API npm start" + ], + "file": "${input:dotEnvFilePath}" + }, + { + "label": "Restore API", + "type": "shell", + "command": "azd restore --service api", + "presentation": { + "reveal": "silent" + }, + "problemMatcher": [] + }, + { + "label": "API npm start", + "detail": "Helper task--use 'Start API' task to ensure environment is set up correctly", + "type": "shell", + "command": "npm run start", + "options": { + "cwd": "${workspaceFolder}/src/api/", + "env": { + "NODE_ENV": "development" + } + }, + "presentation": { + "panel": "dedicated", + }, + "problemMatcher": [] + }, + + { + "label": "Start API and Web", + "dependsOn":[ + "Start API", + "Start Web" + ], + "problemMatcher": [] + } + ], + + "inputs": [ + { + "id": "dotEnvFilePath", + "type": "command", + "command": "azure-dev.commands.getDotEnvFilePath" + } + ] +} diff --git a/templates/todo/projects/nodejs-mongo-aks/README.md b/templates/todo/projects/nodejs-mongo-aks/README.md new file mode 100644 index 00000000000..59112b9636b --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/README.md @@ -0,0 +1,210 @@ +# ToDo Application with a Node.js API and Azure Cosmos DB API for MongoDB on Azure App Service + +[![Open in Remote - Containers](https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/todo-nodejs-mongo) + +A complete ToDo application that includes everything you need to build, deploy, and monitor an Azure solution. This application uses the Azure Developer CLI (azd) to get you up and running on Azure quickly, React.js for the Web application, Node.js for the API, Azure Cosmos DB API for MongoDB for storage, and Azure Monitor for monitoring and logging. It includes application code, tools, and pipelines that serve as a foundation from which you can build upon and customize when creating your own solutions. + +Let's jump in and get the ToDo app up and running in Azure. When you are finished, you will have a fully functional web app deployed on Azure. In later steps, you'll see how to setup a pipeline and monitor the application. + +Screenshot of deployed ToDo app + +Screenshot of the deployed ToDo app + +### Prerequisites + +The following prerequisites are required to use this application. Please ensure that you have them all installed locally. + +- [Azure Developer CLI](https://aka.ms/azure-dev/install) + - Windows: + ```powershell + powershell -ex AllSigned -c "Invoke-RestMethod 'https://aka.ms/install-azd.ps1' | Invoke-Expression" + ``` + - Linux/MacOS: + ``` + curl -fsSL https://aka.ms/install-azd.sh | bash + ``` +- [Azure CLI (2.38.0+)](https://docs.microsoft.com/cli/azure/install-azure-cli) +- [Node.js with npm (16.13.1+)](https://nodejs.org/) - for API backend and Web frontend +- [Git (2.36.1+)](https://git-scm.com/) +PLACEHOLDERIACTOOLS + +### Quickstart + +The fastest way for you to get this application up and running on Azure is to use the `azd up` command. This single command will create and configure all necessary Azure resources - including access policies and roles for your account and service-to-service communication with Managed Identities. + +1. Open a terminal, create a new empty folder, and change into it. +1. Run the following command to initialize the project, provision Azure resources, and deploy the application code. + +```bash +azd up --template todo-nodejs-mongo +``` + +You will be prompted for the following information: + +- `Environment Name`: This will be used as a prefix for the resource group that will be created to hold all Azure resources. This name should be unique within your Azure subscription. +- `Azure Location`: The Azure location where your resources will be deployed. +- `Azure Subscription`: The Azure Subscription where your resources will be deployed. + +> NOTE: This may take a while to complete as it executes three commands: `azd init` (initializes environment), `azd provision` (provisions Azure resources), and `azd deploy` (deploys application code). You will see a progress indicator as it provisions and deploys your application. + +When `azd up` is complete it will output the following URLs: + +- Azure Portal link to view resources +- ToDo Web application frontend +- ToDo API application + +!["azd up output"](assets/urls.png) + +Click the web application URL to launch the ToDo app. Create a new collection and add some items. This will create monitoring activity in the application that you will be able to see later when you run `azd monitor`. + +> NOTE: +> +> - The `azd up` command will create Azure resources that will incur costs to your Azure subscription. You can clean up those resources manually via the Azure portal or with the `azd down` command. +> - You can call `azd up` as many times as you like to both provision and deploy your solution, but you only need to provide the `--template` parameter the first time you call it to get the code locally. Subsequent `azd up` calls do not require the template parameter. If you do provide the parameter, all your local source code will be overwritten if you agree to overwrite when prompted. +> - You can always create a new environment with `azd env new`. + +### Application Architecture + +This application utilizes the following Azure resources: + +- [**Azure App Services**](https://docs.microsoft.com/azure/app-service/) to host the Web frontend and API backend +- [**Azure Cosmos DB API for MongoDB**](https://docs.microsoft.com/azure/cosmos-db/mongodb/mongodb-introduction) for storage +- [**Azure Monitor**](https://docs.microsoft.com/azure/azure-monitor/) for monitoring and logging +- [**Azure Key Vault**](https://docs.microsoft.com/azure/key-vault/) for securing secrets + +Here's a high level architecture diagram that illustrates these components. Notice that these are all contained within a single [resource group](https://docs.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal), that will be created for you when you create the resources. + +Application architecture diagram + +> This template provisions resources to an Azure subscription that you will select upon provisioning them. Please refer to the [Pricing calculator for Microsoft Azure](https://azure.microsoft.com/pricing/calculator/) and, if needed, update the included Azure resource definitions found in `infra/main.bicep` to suit your needs. + +### Application Code + +The repo is structured to follow the [Azure Developer CLI](https://aka.ms/azure-dev/overview) conventions including: + +- **Source Code**: All application source code is located in the `src` folder. +- **Infrastructure as Code**: All application "infrastructure as code" files are located in the `infra` folder. +- **Azure Developer Configuration**: An `azure.yaml` file located in the root that ties the application source code to the Azure services defined in your "infrastructure as code" files. +- **GitHub Actions**: A sample GitHub action file is located in the `.github/workflows` folder. +- **VS Code Configuration**: All VS Code configuration to run and debug the application is located in the `.vscode` folder. + +### Azure Subscription + +This template will create infrastructure and deploy code to Azure. If you don't have an Azure Subscription, you can sign up for a [free account here](https://azure.microsoft.com/free/). + +### Azure Developer CLI - VS Code Extension + +The Azure Developer experience includes an Azure Developer CLI VS Code Extension that mirrors all of the Azure Developer CLI commands into the `azure.yaml` context menu and command palette options. If you are a VS Code user, then we highly recommend installing this extension for the best experience. + +Here's how to install it: + +#### VS Code + +1. Click on the "Extensions" tab in VS Code +1. Search for "Azure Developer CLI" - authored by Microsoft +1. Click "Install" + +#### Marketplace + +1. Go to the [Azure Developer CLI - VS Code Extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.azure-dev) page +1. Click "Install" + +Once the extension is installed, you can press `F1`, and type "Azure Developer CLI" to see all of your available options. You can also right click on your project's `azure.yaml` file for a list of commands. + +### Next Steps + +At this point, you have a complete application deployed on Azure. But there is much more that the Azure Developer CLI can do. These next steps will introduce you to additional commands that will make creating applications on Azure much easier. Using the Azure Developer CLI, you can setup your pipelines, monitor your application, test and debug locally. + +#### Set up a pipeline using `azd pipeline` + +This template includes a GitHub Actions pipeline configuration file that will deploy your application whenever code is pushed to the main branch. You can find that pipeline file here: `.github/workflows`. + +Setting up this pipeline requires you to give GitHub permission to deploy to Azure on your behalf, which is done via a Service Principal stored in a GitHub secret named `AZURE_CREDENTIALS`. The `azd pipeline config` command will automatically create a service principal for you. The command also helps to create a private GitHub repository and pushes code to the newly created repo. + +Before you call the `azd pipeline config` command, you'll need to install the following: + +- [GitHub CLI (2.3+)](https://github.com/cli/cli) + +Run the following command to set up a GitHub Action: + +```bash +azd pipeline config +``` + +> Support for Azure DevOps Pipelines is coming soon to `azd pipeline config`. In the meantime, you can follow the instructions found here: [.azdo/pipelines/README.md](./.azdo/pipelines/README.md) to set it up manually. + +#### Monitor the application using `azd monitor` + +To help with monitoring applications, the Azure Dev CLI provides a `monitor` command to help you get to the various Application Insights dashboards. + +- Run the following command to open the "Overview" dashboard: + + ```bash + azd monitor --overview + ``` + +- Live Metrics Dashboard + + Run the following command to open the "Live Metrics" dashboard: + + ```bash + azd monitor --live + ``` + +- Logs Dashboard + + Run the following command to open the "Logs" dashboard: + + ```bash + azd monitor --logs + ``` + +#### Run and Debug Locally + +The easiest way to run and debug is to leverage the Azure Developer CLI Visual Studio Code Extension. Refer to this [walk-through](https://aka.ms/azure-dev/vscode) for more details. + +#### Clean up resources + +When you are done, you can delete all the Azure resources created with this template by running the following command: + +```bash +azd down +``` + +### Additional azd commands + +The Azure Developer CLI includes many other commands to help with your Azure development experience. You can view these commands at the terminal by running `azd help`. You can also view the full list of commands on our [Azure Developer CLI command](https://aka.ms/azure-dev/ref) page. + +## Troubleshooting/Known issues + +Sometimes, things go awry. If you happen to run into issues, then please review our ["Known Issues"](https://aka.ms/azure-dev/knownissues) page for help. If you continue to have issues, then please file an issue in our main [Azure Dev](https://aka.ms/azure-dev/issues) repository. + +## Security + +### Roles + +This template creates a [managed identity](https://docs.microsoft.com/azure/active-directory/managed-identities-azure-resources/overview) for your app inside your Azure Active Directory tenant, and it is used to authenticate your app with Azure and other services that support Azure AD authentication like Key Vault via access policies. You will see principalId referenced in the infrastructure as code files, that refers to the id of the currently logged in Azure CLI user, which will be granted access policies and permissions to run the application locally. To view your managed identity in the Azure Portal, follow these [steps](https://docs.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-to-view-managed-identity-service-principal-portal). + +### Key Vault + +This template uses [Azure Key Vault](https://docs.microsoft.com/azure/key-vault/general/overview) to securely store your Cosmos DB connection string for the provisioned Cosmos DB account. Key Vault is a cloud service for securely storing and accessing secrets (API keys, passwords, certificates, cryptographic keys) and makes it simple to give other Azure services access to them. As you continue developing your solution, you may add as many secrets to your Key Vault as you require. + +## Uninstall + +To uninstall the Azure Developer CLI: + +Windows: + +``` +powershell -ex AllSigned -c "Invoke-RestMethod 'https://aka.ms/uninstall-azd.ps1' | Invoke-Expression" +``` + +Linux/MacOS: + +``` +curl -fsSL https://aka.ms/uninstall-azd.sh | bash +``` + +## Reporting Issues and Feedback + +If you have any feature requests, issues, or areas for improvement, please [file an issue](https://aka.ms/azure-dev/issues). To keep up-to-date, ask questions, or share suggestions, join our [GitHub Discussions](https://aka.ms/azure-dev/discussions). You may also contact us via AzDevTeam@microsoft.com. diff --git a/templates/todo/projects/nodejs-mongo-aks/assets/resources.png b/templates/todo/projects/nodejs-mongo-aks/assets/resources.png new file mode 100644 index 00000000000..07f06cb01a6 Binary files /dev/null and b/templates/todo/projects/nodejs-mongo-aks/assets/resources.png differ diff --git a/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/deployment.yaml b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/deployment.yaml new file mode 100644 index 00000000000..ae5c5996154 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: todo-api +spec: + replicas: 1 + selector: + matchLabels: + app: todo-api + template: + metadata: + labels: + app: todo-api + spec: + containers: + - name: todo-api + image: crppnnadqdq7owqmxjqvwcid425o.azurecr.io/todo-nodejs-mongo-aks/api:latest + ports: + - containerPort: 3100 + env: + - name: AZURE_CLIENT_ID + value: 8111defe-2f69-4893-b025-ffeaea359c4a + - name: AZURE_KEY_VAULT_ENDPOINT + valueFrom: + secretKeyRef: + name: azd + key: AZURE_KEY_VAULT_ENDPOINT + optional: false + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: azd + key: APPLICATIONINSIGHTS_CONNECTION_STRING + optional: false diff --git a/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/ingress.yaml b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/ingress.yaml new file mode 100644 index 00000000000..c2baf8239ac --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: todo-ingress-api + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: webapprouting.kubernetes.azure.com + rules: + - http: + paths: + - path: /api(/|$)(.*) + pathType: Prefix + backend: + service: + name: todo-api + port: + number: 80 diff --git a/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/service.yaml b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/service.yaml new file mode 100644 index 00000000000..a1622390d2b --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/api/manifests/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: todo-api +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 3100 + selector: + app: todo-api diff --git a/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/deployment.yaml b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/deployment.yaml new file mode 100644 index 00000000000..512d0a80443 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: todo-web +spec: + replicas: 1 + selector: + matchLabels: + app: todo-web + template: + metadata: + labels: + app: todo-web + spec: + containers: + - name: todo-web + image: crppnnadqdq7owqmxjqvwcid425o.azurecr.io/todo-nodejs-mongo-aks/web:latest + ports: + - containerPort: 3000 + env: + - name: REACT_APP_API_BASE_URL + value: /api + - name: REACT_APP_APPLICATIONINSIGHTS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: azd + key: APPLICATIONINSIGHTS_CONNECTION_STRING + optional: false diff --git a/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/ingress.yaml b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/ingress.yaml new file mode 100644 index 00000000000..5676d5a9e01 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: todo-ingress-web +spec: + ingressClassName: webapprouting.kubernetes.azure.com + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: todo-web + port: + number: 80 diff --git a/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/service.yaml b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/service.yaml new file mode 100644 index 00000000000..ff91f4ba4b9 --- /dev/null +++ b/templates/todo/projects/nodejs-mongo-aks/src/web/manifests/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: todo-web +spec: + type: ClusterIP + ports: + - port: 80 + selector: + app: todo-web