diff --git a/infra/modules/providers/azure/data-factory/README.md b/infra/modules/providers/azure/data-factory/README.md
index fd47441c..ef528613 100644
--- a/infra/modules/providers/azure/data-factory/README.md
+++ b/infra/modules/providers/azure/data-factory/README.md
@@ -17,15 +17,13 @@ An instance of the `data-factory` module deploys the _**Data Factory**_ in order
- Ability to provision a single Data Factory instance
- Ability to provision a configurable Pipeline
- Ability to configure Trigger
-- Ability to configure SQL server Dataset
-- Ability to configure SQL server Linked Service
+
## Out Of Scope
The following are not support in the time being
- Creating Multiple pipelines
-- Only SQL server Dataset/Linked Service are implemented.
## Definition
@@ -35,8 +33,6 @@ Terraform resources used to define the `data-factory` module include the followi
- [azurerm_data_factory_integration_runtime_managed](https://www.terraform.io/docs/providers/azurerm/r/data_factory_integration_runtime_managed.html)
- [azurerm_data_factory_pipeline](https://www.terraform.io/docs/providers/azurerm/r/data_factory_pipeline.html)
- [azurerm_data_factory_trigger_schedule](https://www.terraform.io/docs/providers/azurerm/r/data_factory_trigger_schedule.html)
-- [azurerm_data_factory_dataset_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_dataset_sql_server_table.html)
-- [azurerm_data_factory_linked_service_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_linked_service_sql_server.html)
## Usage
@@ -60,11 +56,6 @@ module "data_factory" {
data_factory_trigger_name = "adftrigger"
data_factory_trigger_interval = 1
data_factory_trigger_frequency = "Minute"
- data_factory_dataset_sql_name = "adfsqldataset"
- data_factory_dataset_sql_table_name = "adfsqldatasettable"
- data_factory_dataset_sql_folder = ""
- data_factory_linked_sql_name = "adfsqllinked"
- data_factory_linked_sql_connection_string = "Server=tcp:adfsql..."
}
```
diff --git a/infra/modules/providers/azure/data-factory/datasets.tf b/infra/modules/providers/azure/data-factory/datasets.tf
deleted file mode 100644
index 8133ddba..00000000
--- a/infra/modules/providers/azure/data-factory/datasets.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-
-resource "azurerm_data_factory_dataset_sql_server_table" "main" {
- name = var.data_factory_dataset_sql_name
- resource_group_name = data.azurerm_resource_group.main.name
- data_factory_name = azurerm_data_factory.main.name
- linked_service_name = azurerm_data_factory_linked_service_sql_server.main.name
- table_name = var.data_factory_dataset_sql_table_name
-}
\ No newline at end of file
diff --git a/infra/modules/providers/azure/data-factory/linkedservices.tf b/infra/modules/providers/azure/data-factory/linkedservices.tf
deleted file mode 100644
index 1c4d5c60..00000000
--- a/infra/modules/providers/azure/data-factory/linkedservices.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-
-resource "azurerm_data_factory_linked_service_sql_server" "main" {
- name = var.data_factory_linked_sql_name
- resource_group_name = data.azurerm_resource_group.main.name
- data_factory_name = azurerm_data_factory.main.name
- connection_string = var.data_factory_linked_sql_connection_string
- integration_runtime_name = azurerm_data_factory_integration_runtime_managed.main.name
-}
diff --git a/infra/modules/providers/azure/data-factory/output.tf b/infra/modules/providers/azure/data-factory/output.tf
index 41183a80..c51ba94a 100644
--- a/infra/modules/providers/azure/data-factory/output.tf
+++ b/infra/modules/providers/azure/data-factory/output.tf
@@ -28,16 +28,6 @@ output "trigger_interval" {
value = azurerm_data_factory_trigger_schedule.main.interval
}
-output "sql_dataset_id" {
- description = "The ID of the SQL server dataset created"
- value = azurerm_data_factory_dataset_sql_server_table.main.id
-}
-
-output "sql_linked_service_id" {
- description = "The ID of the SQL server Linked service created"
- value = azurerm_data_factory_linked_service_sql_server.main.id
-}
-
output "adf_identity_principal_id" {
description = "The ID of the principal(client) in Azure active directory"
value = azurerm_data_factory.main.identity[0].principal_id
diff --git a/infra/modules/providers/azure/data-factory/terraform.tfvars.template b/infra/modules/providers/azure/data-factory/terraform.tfvars.template
index 84048298..8bbae88c 100644
--- a/infra/modules/providers/azure/data-factory/terraform.tfvars.template
+++ b/infra/modules/providers/azure/data-factory/terraform.tfvars.template
@@ -2,10 +2,7 @@ resource_group_name = ""
data_factory_name = ""
data_factory_runtime_name = ""
data_factory_pipeline_name = ""
-data_factory_dataset_sql_name = ""
-data_factory_dataset_sql_table_name = ""
-data_factory_linked_sql_name = ""
-data_factory_linked_sql_connection_string = ""
+data_factory_trigger_name = ""
vnet_integration = {
vnet_id = ""
subnet_name = ""
diff --git a/infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go b/infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go
index 66c11b36..19efe05b 100644
--- a/infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go
+++ b/infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go
@@ -25,16 +25,6 @@ func TestDataFactory(t *testing.T) {
"data_factory_name",
"pipeline_name",
),
- VerifyCreatedDataset(subscription,
- "resource_group_name",
- "data_factory_name",
- "sql_dataset_id",
- ),
- VerifyCreatedLinkedService(subscription,
- "resource_group_name",
- "data_factory_name",
- "sql_linked_service_id",
- ),
},
}
integration.RunIntegrationTests(&testFixture)
diff --git a/infra/modules/providers/azure/data-factory/tests/test.tfvars b/infra/modules/providers/azure/data-factory/tests/test.tfvars
index e2166811..8bbae88c 100644
--- a/infra/modules/providers/azure/data-factory/tests/test.tfvars
+++ b/infra/modules/providers/azure/data-factory/tests/test.tfvars
@@ -1,13 +1,9 @@
-resource_group_name = "adftest"
-data_factory_name = "adftest"
-data_factory_runtime_name = "adfrttest"
-data_factory_pipeline_name = "testpipeline"
-data_factory_trigger_name = "testtrigger"
-data_factory_dataset_sql_name = "testsql"
-data_factory_dataset_sql_table_name = "adfsqltableheba"
-data_factory_linked_sql_name = "testlinkedsql"
-data_factory_linked_sql_connection_string = "connectionstring"
+resource_group_name = ""
+data_factory_name = ""
+data_factory_runtime_name = ""
+data_factory_pipeline_name = ""
+data_factory_trigger_name = ""
vnet_integration = {
- vnet_id = "/subscriptions/resourceGroups/providers/Microsoft.Network/virtualNetworks/testvnet"
- subnet_name = "default"
+ vnet_id = ""
+ subnet_name = ""
}
diff --git a/infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go b/infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go
index c6dd0a0f..78bf3bad 100644
--- a/infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go
+++ b/infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go
@@ -1,24 +1,11 @@
package unit
import (
- "encoding/json"
- "strings"
"testing"
-
- "github.com/gruntwork-io/terratest/modules/random"
tests "github.com/microsoft/cobalt/infra/modules/providers/azure/data-factory/tests"
"github.com/microsoft/terratest-abstraction/unit"
)
-// helper function to parse blocks of JSON into a generic Go map
-func asMap(t *testing.T, jsonString string) map[string]interface{} {
- var theMap map[string]interface{}
- if err := json.Unmarshal([]byte(jsonString), &theMap); err != nil {
- t.Fatal(err)
- }
- return theMap
-}
-
func TestTemplate(t *testing.T) {
expectedDataFactory := map[string]interface{}{
@@ -53,27 +40,16 @@ func TestTemplate(t *testing.T) {
"frequency": "Minute",
}
- expectedDatasetSQL := map[string]interface{}{
- "name": "testsql",
- }
-
- expectedLinkedSQL := map[string]interface{}{
- "name": "testlinkedsql",
- "connection_string": "connectionstring",
- }
-
testFixture := unit.UnitTestFixture{
GoTest: t,
TfOptions: tests.DataFactoryTFOptions,
PlanAssertions: nil,
- ExpectedResourceCount: 6,
+ ExpectedResourceCount: 4,
ExpectedResourceAttributeValues: unit.ResourceDescription{
"azurerm_data_factory.main": expectedDataFactory,
"azurerm_data_factory_integration_runtime_managed.main": expectedDFIntRunTime,
"azurerm_data_factory_pipeline.main": expectedPipeline,
"azurerm_data_factory_trigger_schedule.main": expectedTrigger,
- "azurerm_data_factory_dataset_sql_server_table.main": expectedDatasetSQL,
- "azurerm_data_factory_linked_service_sql_server.main": expectedLinkedSQL,
},
}
diff --git a/infra/modules/providers/azure/data-factory/variables.tf b/infra/modules/providers/azure/data-factory/variables.tf
index 96ab0679..e6c38bef 100644
--- a/infra/modules/providers/azure/data-factory/variables.tf
+++ b/infra/modules/providers/azure/data-factory/variables.tf
@@ -68,34 +68,4 @@ variable "data_factory_trigger_frequency" {
description = "The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute."
type = string
default = "Minute"
-}
-
-variable "data_factory_dataset_sql_name" {
- description = "Specifies the name of the Data Factory Dataset SQL Server Table. Only letters, numbers and '_' are allowed."
- type = string
- default = ""
-}
-
-variable "data_factory_dataset_sql_table_name" {
- description = "The table name of the Data Factory Dataset SQL Server Table."
- type = string
- default = ""
-}
-
-variable "data_factory_dataset_sql_folder" {
- description = "The folder that this Dataset is in. If not specified, the Dataset will appear at the root level."
- type = string
- default = ""
-}
-
-variable "data_factory_linked_sql_name" {
- description = "Specifies the name of the Data Factory Linked Service SQL Server. Changing this forces a new resource to be created."
- type = string
- default = ""
-}
-
-variable "data_factory_linked_sql_connection_string" {
- description = "The connection string in which to authenticate with the SQL Server."
- type = string
- default = ""
}
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/.env.template b/infra/templates/az-svc-data-integration-mlw/.env.template
new file mode 100644
index 00000000..f93d1f90
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/.env.template
@@ -0,0 +1,12 @@
+export ARM_ACCESS_KEY=
+export ARM_CLIENT_ID=
+export ARM_CLIENT_SECRET=
+export ARM_SUBSCRIPTION_ID=
+export ARM_TENANT_ID=
+export BUILD_BUILDID=1
+export GO_VERSION=1.12.5
+export TF_VAR_remote_state_account=
+export TF_VAR_remote_state_container=
+export TF_VERSION=0.12.4
+export TF_WARN_OUTPUT_ERRORS=1
+export TF_VAR_resource_group_location=eastus
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/README.md b/infra/templates/az-svc-data-integration-mlw/README.md
new file mode 100644
index 00000000..f088af57
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/README.md
@@ -0,0 +1,139 @@
+# Azure Application Services
+
+The `az-svc-data-integration-mlw` template is intended to be a reference for running a set of app services.
+
+
+## Use-Case
+
+This particular template creates an Azure environment with a small set of fully managed microservices.
+
+
+## Scenarios this template should avoid
+
+This template is an adequate solution where the service count is less than 10. For Azure customers interested with provisioning more than 10 services, we recommend using AKS. Reason being that with Kubernetes you can maximize cluster node CPU cores which helps minimize cloud resourcing costs.
+
+## Technical Design
+Template design [specifications](docs/design/README.md).
+
+## Architecture
+
+
+
+## Prerequisites
+
+1. Azure Subscription
+2. An available Service Principal with API Permissions granted with Admin Consent within Azure app registration. The required Azure Active Directory Graph app role is `Application.ReadWrite.OwnedBy`
+
+
+
+3. Terraform and Go are locally installed
+4. Azure Storage Account is [setup](https://docs.microsoft.com/en-us/azure/terraform/terraform-backend) to store Terraform state
+5. Set up your Local environment variables by creating a `.env` file that contains the following information:
+
+```
+ARM_SUBSCRIPTION_ID="<az-service-principal-subscription-id>"
+ARM_CLIENT_ID="<az-service-principal-client-id>"
+ARM_CLIENT_SECRET="<az-service-principal-auth-secret>"
+ARM_TENANT_ID="<az-service-principal-tenant>"
+ARM_ACCESS_KEY="<remote-state-storage-account-primary-key>"
+TF_VAR_remote_state_account="<tf-remote-state-storage-account-name>"
+TF_VAR_remote_state_container="<tf-remote-state-storage-container-name>"
+```
+
+## Cost
+
+Azure environment cost ballpark [estimate](https://azure.com/e/92b05a7cd1e646368ab74772e3122500). This is subject to change and is driven from the resource pricing tiers configured when the template is deployed.
+
+## Deployment Steps
+
+1. Execute the following commands to set up your local environment variables:
+
+*Note for Windows Users using WSL*: We recommend running dos2unix utility on the environment file via `dos2unix .env` prior to sourcing your environment variables to chop trailing newline and carriage return characters.
+
+```bash
+# these commands setup all the environment variables needed to run this template
+DOT_ENV=<path to your .env file>
+export $(cat $DOT_ENV | xargs)
+```
+
+2. Execute the following command to configure your local Azure CLI.
+
+```bash
+# This logs your local Azure CLI in using the configured service principal.
+az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
+```
+
+3. Navigate to the `terraform.tfvars` terraform file. Here's a sample of the terraform.tfvars file for this template.
+
+```HCL
+resource_group_location = "centralus"
+prefix = "test-services"
+
+# Targets that will be configured to also setup AuthN with Easy Auth
+app_services = [
+ {
+ app_name = "tf-test-svc-1"
+ image = null
+ app_settings = {
+ "one_sweet_app_setting" = "brilliant"
+ }
+ },
+ {
+ app_name = "tf-test-svc-2"
+ image = null
+ app_settings = {
+ "another_sweet_svc_app_setting" = "ok"
+ }
+ }
+]
+```
+
+4. Execute the following commands to set up your terraform workspace.
+
+```bash
+# This configures terraform to leverage a remote backend that will help you and your
+# team keep consistent state
+terraform init -backend-config "storage_account_name=${TF_VAR_remote_state_account}" -backend-config "container_name=${TF_VAR_remote_state_container}"
+
+# This command configures terraform to use a workspace unique to you. This allows you to work
+# without stepping over your teammate's deployments
+TF_WORKSPACE="az-micro-svc-$USER"
+terraform workspace new $TF_WORKSPACE || terraform workspace select $TF_WORKSPACE
+```
+
+5. Execute the following commands to orchestrate a deployment.
+
+```bash
+# See what terraform will try to deploy without actually deploying
+terraform plan
+
+# Execute a deployment
+terraform apply
+```
+
+6. Optionally execute the following command to teardown your deployment and delete your resources.
+
+```bash
+# Destroy resources and tear down deployment. Only do this if you want to destroy your deployment.
+terraform destroy
+```
+
+## Automated Testing
+
+### Unit Testing
+
+Navigate to the template folder `infra/templates/az-svc-data-integration-mlw`. Unit tests can be run using the following command:
+
+```
+go test -v $(go list ./... | grep "unit")
+```
+
+### Integration Testing
+
+Please confirm that you've completed the `terraform apply` step before running the integration tests as we're validating the active terraform workspace.
+
+Integration tests can be run using the following command:
+
+```
+go test -v $(go list ./... | grep "integration")
+```
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/app.tf b/infra/templates/az-svc-data-integration-mlw/app.tf
new file mode 100644
index 00000000..d1bc0530
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/app.tf
@@ -0,0 +1,126 @@
+resource "azurerm_resource_group" "app_rg" {
+ name = local.app_rg_name
+ location = local.region
+}
+
+# Note: this should be uncommented for production scenarios. It is commented
+# to support a teardown after deployment for the CICD pipeline.
+# resource "azurerm_management_lock" "app_rg_lock" {
+# name = local.app_rg_lock
+# scope = azurerm_resource_group.app_rg.id
+# lock_level = "CanNotDelete"
+
+# lifecycle {
+# prevent_destroy = true
+# }
+# }
+
+module "network" {
+ source = "../../modules/providers/azure/network"
+ vnet_name = local.vnet_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ address_space = var.address_space
+ subnets = local.subnets
+}
+
+module "container_registry" {
+ source = "../../modules/providers/azure/container-registry"
+ container_registry_name = local.acr_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ container_registry_admin_enabled = false
+ // Note: only premium ACRs allow configuration of network access restrictions
+ container_registry_sku = var.container_registry_sku
+ subnet_id_whitelist = module.network.subnet_ids
+}
+
+module "app_insights" {
+ source = "../../modules/providers/azure/app-insights"
+ service_plan_resource_group_name = azurerm_resource_group.app_rg.name
+ appinsights_name = local.ai_name
+ appinsights_application_type = "web"
+}
+
+
+module "func_app_service_plan" {
+ source = "../../modules/providers/azure/service-plan"
+ resource_group_name = azurerm_resource_group.app_rg.name
+ service_plan_name = local.func_app_sp_name
+ # scaling_rules = var.scaling_rules
+ service_plan_tier = var.func_app_service_plan_tier
+ service_plan_size = var.func_app_service_plan_size
+ service_plan_kind = var.func_app_service_plan_kind
+ service_plan_reserved = var.func_app_service_plan_reserved
+}
+
+
+
+
+module "app_monitoring" {
+ source = "../../modules/providers/azure/app-monitoring"
+ resource_group_name = azurerm_resource_group.app_rg.name
+ resource_ids = [module.func_app_service_plan.app_service_plan_id]
+ action_group_name = var.action_group_name
+ action_group_email_receiver = var.action_group_email_receiver
+ metric_alert_name = var.metric_alert_name
+ metric_alert_frequency = var.metric_alert_frequency
+ metric_alert_period = var.metric_alert_period
+ metric_alert_criteria_namespace = var.metric_alert_criteria_namespace
+ metric_alert_criteria_name = var.metric_alert_criteria_name
+ metric_alert_criteria_aggregation = var.metric_alert_criteria_aggregation
+ metric_alert_criteria_operator = var.metric_alert_criteria_operator
+ metric_alert_criteria_threshold = var.metric_alert_criteria_threshold
+ monitoring_dimension_values = var.monitoring_dimension_values
+}
+
+resource "azurerm_resource_group" "mlw_rg" {
+ name = local.mlw_rg_name
+ location = local.region
+}
+
+module "mlw_app_insights" {
+ source = "../../modules/providers/azure/app-insights"
+ service_plan_resource_group_name = azurerm_resource_group.mlw_rg.name
+ appinsights_name = local.mlw_ai_name
+ appinsights_application_type = "web"
+}
+
+module "ml_workspace" {
+ source = "../../modules/providers/azure/ml-workspace"
+ name = local.mlw_name
+ resource_group_name = azurerm_resource_group.mlw_rg.name
+ application_insights_id = module.mlw_app_insights.id
+ key_vault_id = module.keyvault.keyvault_id
+ storage_account_id = module.sys_storage_account.id
+ sku_name = var.sku_name
+}
+
+module "function_app" {
+ source = "../../modules/providers/azure/function-app"
+ fn_name_prefix = local.func_app_name_prefix
+ resource_group_name = azurerm_resource_group.app_rg.name
+ service_plan_id = module.func_app_service_plan.app_service_plan_id
+ storage_account_resource_group_name = module.sys_storage_account.resource_group_name
+ storage_account_name = module.sys_storage_account.name
+ vnet_subnet_id = module.network.subnet_ids[0]
+ fn_app_settings = local.func_app_settings
+ fn_app_config = var.fn_app_config
+}
+
+
+module "data_factory" {
+ source = "../../modules/providers/azure/data-factory"
+ data_factory_name = local.data_factory_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ data_factory_runtime_name = local.data_factory_runtime_name
+ number_of_nodes = var.adf_number_of_nodes
+ max_parallel_executions_per_node = var.adf_max_parallel_executions_per_node
+ vnet_integration = {
+ vnet_id = module.network.subnet_ids[0]
+ subnet_name = local.subnet_name
+ }
+ data_factory_pipeline_name = local.data_factory_pipeline_name
+ data_factory_trigger_name = local.data_factory_trigger_name
+ data_factory_trigger_interval = var.adf_trigger_interval
+ data_factory_trigger_frequency = var.adf_trigger_frequency
+}
+
diff --git a/infra/templates/az-svc-data-integration-mlw/backend.tf b/infra/templates/az-svc-data-integration-mlw/backend.tf
new file mode 100644
index 00000000..96627544
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/backend.tf
@@ -0,0 +1,10 @@
+terraform {
+ backend "azurerm" {
+ key = "terraform.tfstate"
+ }
+}
+
+provider "azurerm" {
+ version = "~>2.6.0"
+ features {}
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/commons.tf b/infra/templates/az-svc-data-integration-mlw/commons.tf
new file mode 100644
index 00000000..4137c63e
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/commons.tf
@@ -0,0 +1,131 @@
+module "provider" {
+ source = "../../modules/providers/azure/provider"
+}
+
+data "azurerm_subscription" "current" {}
+
+data "azurerm_client_config" "current" {}
+
+resource "random_string" "workspace_scope" {
+ keepers = {
+ # Generate a new id each time we switch to a new workspace or app id
+ ws_name = replace(trimspace(lower(terraform.workspace)), "_", "-")
+ app_id = replace(trimspace(lower(var.prefix)), "_", "-")
+ }
+
+ length = max(1, var.randomization_level) // error for zero-length
+ special = false
+ upper = false
+}
+
+locals {
+ // sanitize names
+ resource_prefix = replace(format("%s-%s", trimspace(lower(terraform.workspace)), random_string.workspace_scope.result), "_", "-")
+ app_id = random_string.workspace_scope.keepers.app_id
+ region = replace(trimspace(lower(var.resource_group_location)), "_", "-")
+ ws_name = random_string.workspace_scope.keepers.ws_name
+ suffix = var.randomization_level > 0 ? "-${random_string.workspace_scope.result}" : ""
+ base_storage_name = "${substr(replace(local.resource_prefix, "-", ""), 0, 10)}st" // storage account
+ sys_storage_name = "ml${local.base_storage_name}" //mlops storage account
+ app_storage_name = "dp${local.base_storage_name}" //dataprep storage account
+
+ // base name for resources, name constraints documented here: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions
+ base_name = length(local.app_id) > 0 ? "${local.ws_name}${local.suffix}-${local.app_id}" : "${local.ws_name}${local.suffix}"
+ base_name_21 = length(local.base_name) < 22 ? local.base_name : "${substr(local.base_name, 0, 21 - length(local.suffix))}${local.suffix}"
+ base_name_46 = length(local.base_name) < 47 ? local.base_name : "${substr(local.base_name, 0, 46 - length(local.suffix))}${local.suffix}"
+ base_name_60 = length(local.base_name) < 61 ? local.base_name : "${substr(local.base_name, 0, 60 - length(local.suffix))}${local.suffix}"
+ base_name_70 = length(local.base_name) < 70 ? local.base_name : "${substr(local.base_name, 0, 70 - length(local.suffix))}${local.suffix}"
+ base_name_76 = length(local.base_name) < 77 ? local.base_name : "${substr(local.base_name, 0, 76 - length(local.suffix))}${local.suffix}"
+ base_name_83 = length(local.base_name) < 84 ? local.base_name : "${substr(local.base_name, 0, 83 - length(local.suffix))}${local.suffix}"
+
+ tenant_id = data.azurerm_client_config.current.tenant_id
+
+ // Resource names
+ app_rg_name = "${local.base_name_83}-app-rg" // app resource group (max 90 chars)
+ admin_rg_lock = "${local.base_name_83}-adm-rg-delete-lock" // management lock to prevent deletes
+ app_rg_lock = "${local.base_name_83}-app-rg-delete-lock" // management lock to prevent deletes
+ func_app_sp_name = "${local.base_name}-sp-fa" // service plan
+ ai_name = "${local.base_name}-ai" // app insights
+ mlw_ai_name = "${local.base_name}-mlw-ai" // ml app insights
+ ad_app_name = "${local.base_name}-ad-app" //service principal
+ ad_app_management_name = "${local.base_name}-ad-app-management"
+ graph_id = "00000003-0000-0000-c000-000000000000" // ID for Microsoft Graph API
+ graph_role_id = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" // ID for User.Read API
+ kv_name = "${local.base_name_21}-kv" // key vault (max 24 chars)
+ acr_name = "${replace(local.base_name_46, "-", "")}acr" // container registry (max 50 chars, alphanumeric *only*)
+ vnet_name = "vnet-${local.base_name_60}" // virtual network (max 64 chars)
+ subnet_name = "snet-${local.base_name_76}" // subnet (max 76 chars)
+ function_subnet_name = "snet-func-${local.base_name_76}" // subnet (max 76 chars)
+ func_subnet_delegation_name = "${local.base_name_21}" // subnet (max 76 chars)
+ mlw_name = "${local.base_name_21}-mlw" // workspace (max 30 chars)
+ mlw_rg_name = "${local.base_name_83}-mlw-rg" // mlw resource group (max 90 chars)
+ public_pip_name = "${local.base_name_76}-ip" // public IP (max 80 chars)
+ svc_princ_name = "${local.base_name}-svc-principal" // service principal
+ acr_svc_princ_name = "${local.base_name}-acr-svc-principal" // container registry service principal
+ app_svc_name_prefix = local.base_name_21
+ auth_svc_name_prefix = "${local.base_name_21}-au"
+ smpl_app_postfix = "app"
+ data_factory_name = "adf-cares-${local.base_name_46}"
+ data_factory_runtime_name = "adf-rt-cares-${local.base_name_46}"
+ data_factory_pipeline_name = "adf-pipeline-cares-${local.base_name_83}"
+ data_factory_trigger_name = "adf-trigger-cares-${local.base_name_83}"
+ func_app_name_prefix = "func-cares-${local.base_name_46}" // Function app name (Max 60 chars)
+ cosmos_account_name = "cosmos-cares-${local.base_name_21}" // Cosmos databse account name (max 44 chars)
+ cosmos_db_name = "cosmos-cares-db-${local.base_name_21}" // Cosmos databse account name (max 44 chars)
+
+ app_service_global_config = {
+ aad_client_id = format(local.app_setting_kv_format, local.output_secret_map.aad-client-id)
+ appinsights_key = format(local.app_setting_kv_format, local.output_secret_map.appinsights-key)
+ smpl_app_endpoint = format("https://%s-%s.azurewebsites.net", local.auth_svc_name_prefix, lower(local.smpl_app_postfix))
+ storage_account = module.sys_storage_account.name
+ storage_account_key = format(local.app_setting_kv_format, local.output_secret_map.sys-storage-account-key)
+ }
+
+ mlw_service_global_config = {
+ mlw-appinsights-key = format(local.app_setting_kv_format, local.output_secret_map.mlw-appinsights-key)
+ storage_account = module.sys_storage_account.name
+ storage_account_key = format(local.app_setting_kv_format, local.output_secret_map.sys-storage-account-key)
+ }
+
+ func_app_settings = {
+ clientId = format(local.app_setting_kv_format, local.output_secret_map.app-sp-username)
+ clientKey = format(local.app_setting_kv_format, local.output_secret_map.app-sp-password)
+ storageKey = format(local.app_setting_kv_format, local.output_secret_map.app-storage-account-key)
+ storageName = module.app_storage_account.name
+ tenantId = format(local.app_setting_kv_format, local.output_secret_map.app-sp-tenant-id)
+ statusPollingInterval = ""
+ waitUntil = ""
+ dataDriftContainer = module.app_storage_account.containers.acidatadriftdata.name
+ APPINSIGHTS_INSTRUMENTATIONKEY = format(local.app_setting_kv_format, local.output_secret_map.appinsights-key)
+ }
+
+ subnets = [
+ {
+ name = local.subnet_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ address_prefix = var.subnet_address_prefix
+ service_endpoints = var.subnet_service_endpoints
+ delegation = {
+ name = local.func_subnet_delegation_name
+ service_delegation = {
+ name = "Microsoft.Web/serverFarms"
+ actions = ["Microsoft.Network/virtualNetworks/subnets/action"]
+ }
+ }
+ },
+ {
+ name = local.function_subnet_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ address_prefix = var.function_subnet_address_prefix
+ service_endpoints = var.func_subnet_service_endpoints
+ delegation = {
+ name = ""
+ service_delegation = {
+ name = "Microsoft.Web/serverFarms"
+ actions = ["Microsoft.Network/virtualNetworks/subnets/action"]
+ }
+ }
+ }
+ ]
+
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/docs/design/README.md b/infra/templates/az-svc-data-integration-mlw/docs/design/README.md
new file mode 100644
index 00000000..fb75d02d
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/docs/design/README.md
@@ -0,0 +1,236 @@
+# Technical Design
+
+This document contains the technical design specs for the Terraform template design and deployment for data integration service with machine learning workspace.
+
+## In Scope
+
+- Identify deployment topology
+- Identify key Terraform templates needed for deployment
+- Identify key Terraform modules needed for deployment
+- Identify gaps in Terraform provider templates
+
+## Out of Scope
+
+- Template (Terraform, ARM) implementation
+- Service Containerization
+
+## Deployment Topology
+
+This graphic shows the targeted deployment topology of this template. The deployment is deployed to a single tenant and subscription. The resources are partitioned to three resource groups to align with the different personas:
+
+- Resource Group 1: contains Application services related resources along with the common resources (i.e. KeyVault, app insights)
+- Resource Group 2: Contains storage resources. The best practice is to separate storage resources (i.e. blob storage, SQL, CosmosDB) for protection from any accidental delete
+- Resource Group 3: Contains machine learning related resources
+
+
+
+## Terraform Template Design
+
+This section shows the infrastructure design. It contains the Azure resources we need according to the architecture diagram. There are some modules that were implemented before and we can re-use them in this project that are shown in the diagram. Moreover, we identified the resources that we need to create modules for.
+
+## The Template Design Diagram
+
+The Diagram below outlines the topology of the terraform templates that will deploy the topology called out above.
+
+
+
+## Example Usage
+
+1. Set the path of your template directory
+2. Call the terraform init, plan, apply commands to initialize the terraform deployment then write and apply the plan,
+
+## Template Inputs
+
+### General Inputs
+
+| name | type | default | description |
+|---|---|---|---|
+| `resource_group_location` | string | | The Azure region where all resources in this template should be created |
+| `prefix` | string | | It serves as an identifier used to construct the names of all resources in this template
+| `randomization_level` | number | | Number of additional random characters to include in resource names to insulate against unexpected resource name collisions |
+
+
+### service_plan
+
+| name | type | default | description |
+|---|---|---|---|
+| `monitoring_dimension_values` | `["*"]` | | Dimensions used to determine service plan scaling |
+| `service_plan_size` | string | S1 | The size of the service plan instance. Valid values are I1, I2, I3 |
+| `service_plan_tier` | string | Standard | The tier under which the service plan is created. Details can be found at https://docs.microsoft.com/en-us/azure/app-service/overview-hosting-plans |
+| `scaling_rules` | list | list | The scaling rules for the app service plan. Schema defined here: https://www.terraform.io/docs/providers/azurerm/r/monitor_autoscale_setting.html#rule. Note, the appropriate resource ID will be auto-inflated by the template |
+
+**Note:** Service plan name is auto generated within the template
+
+
+### app-service
+
+| name | type | default | description |
+|---|---|---|---|
+| `app_service_settings` | list(object) | {} | Map of app settings that will be applied across all provisioned app services |
+| `vnet_subnet_id` | string | | The vnet integration subnet gateway identifier |
+| `auth_suffix` | string | | A name to be appended to all azure ad applications |
+
+### function
+
+| name | type | default | description |
+|---|---|---|---|
+| `app_services` | list(object) | | Descriptions of the app services to be deployed. Service level config settings can be provided with the required attribute app_settings |
+
+### app_monitoring
+
+- rule is a type defined [here](https://www.terraform.io/docs/providers/azurerm/r/monitor_autoscale_setting.html#rule)
+- MetricsConfig is a complex map type that adheres to this schema:
+
+```hcl
+list(map(object({
+ action_group_name = string
+ action_group_email_receiver = string
+ metrics = list(map(object({
+ metric_alert_name = string
+ metric_alert_frequency = string
+ metric_alert_period = string
+ metric_alert_criteria_namespace = string
+ metric_alert_criteria_name = string
+ metric_alert_criteria_aggregation = string
+ metric_alert_criteria_operator = string
+ metric_alert_criteria_threshold = string
+ })))
+})))
+```
+
+### app_insight
+
+| name | type | default | description |
+|---|---|---|---|
+| `appinsights_application_type` | string | | Type of the App Insights Application. Valid values are ios for iOS, java for Java web, MobileCenter for App Center, Node.JS for Node.js, other for General, phone for Windows Phone, store for Windows Store and web for ASP.NET |
+
+### vnet
+
+| name | type | default | description |
+|---|---|---|---|
+| `address_space` | string | list(string) | The address space that is used the virtual network. You can supply more than one address space. Changing this forces a new resource to be created |
+| `address_prefix` | list(list(string))| | The address prefix to use for the subnet |
+
+### keyvault
+
+| name | type | default | description |
+|---|---|---|---|
+| `keyvault_name` | string | | Name of the keyvault to create |
+
+### Keyvault_policy
+
+| name | type | default | description |
+|---|---|---|---|
+| `vault_id` | string | |Specifies the name of the Key Vault resource |
+| `tenant_id` | string | |The Azure Active Directory tenant ID that should be used for authenticating requests to the key vault. Changing this forces a new resource to be created |
+| `object_ids` | list(string) | |The object IDs of a user, service principal or security group in the Azure Active Directory tenant for the vault. The object ID must be unique for the list of access policies. Changing this forces a new resource to be created |
+
+### ml_workspace
+
+| name | type | default | description |
+|---|---|---|---|
+| `ml_workspace_sku_name` | string | "Enterprise" | Specifies the name of the Key Vault resource |
+
+### storage_account
+
+| name | type | default | description |
+|---|---|---|---|
+| `container_names` | list(string) | | The list of storage container names to create. Names must be unique per storage account |
+
+### container_registry
+
+| name | type | default | description |
+|---|---|---|---|
+| `container_registry_sku` | list(object) | "Standard" | The container registry SKU. ie. 'Standard' 'Premium'. |
+
+### data_factory
+
+| name | type | default | description |
+|---|---|---|---|
+| `adf_trigger_name` | string | | Specifies the name of the Data Factory Schedule Trigger. Changing this forces a new resource to be created. Must be globally unique |
+| `pipeline_name ` | string | | The Data Factory Pipeline name that the trigger will act on |
+| `interval` | number | 1 | The interval for how often the trigger occurs. This defaults to 1 |
+| `frequency ` | string | "Minute" | The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute.
+
+### private_endpoint
+
+| name | type | default | description |
+|---|---|---|---|
+| `private_connection_name` | string | | Specifies the Name of the Private Service Connection. Changing this forces a new resource to be created |
+| `is_manual_connection` | boolean | false | Does the Private Endpoint require Manual Approval from the remote resource owner? Changing this forces a new resource to be created |
+| `private_connection_resource_id ` | string | | The ID of the Private Link Enabled Remote Resource which this Private Endpoint should be connected to. Changing this forces a new resource to be created |
+
+## Template Output
+
+| name | type | description |
+|---|---|---|
+| `app_service_config` | map(string) | Map of app service name with their respective slot apps |
+| `app_service_fqdns` | map(string) | Map of app service name to VNET accessible domain name |
+| `app_service_names` | list(string) | List of provisioned app service names |
+| `app_service_ids` | list(string) | The IDs of the app services provisioned |
+| `azuread_app_ids` | list(string) | The AAD application object ids used for app service easy auth |
+| `app_service_msi_object_ids` | list(string) | The app service System Assigned MSI object ids |
+| `service_plan_name` | string | Service Plan Name |
+| `service_plan_id` | string | Service Plan Resource ID |
+| `resource_group` | string | Resource group name for the application specific resource group |
+| `service_principal_application_id` | string | Service principal application ID |
+| `key_vault_endpoint` | string | Key Vault endpoint |
+| `blob_storage_endpoint` | string | Blob Storage endpoint |
+| `sys_storage_account` | string | The name of the ml storage account |
+| `sys_storage_account_id` | string | Resource Identifier of the ml storage account |
+| `sys_storage_account_containers` | string | Map of the ml storage account containers |
+| `app_storage_account_name` | string | Name of Storage Account |
+| `app_storage_account_id` | string | Resource Identifier of the storage account |
+| `app_storage_account_containers` | string | Map of storage account containers |
+| `function_app_properties` | map(string) | Map of function app id and fqdn |
+| `function_app_fqdns` | list(string) | The URLs of the app services created |
+| `rule_resource_id` | string | The id of a metric alert rule |
+| `container_registry_id` | string | The resource identifier of the container registry |
+| `container_registry_name` | string | The name of the container registry. |
+| `contributor_service_principal_id` | string | ID of the service principal with contributor access to provisioned resources |
+| `ml_identity_principal_id` | list(string) | The id of the Azure Machine Learning service managed identity object id |
+| `mlw_dev_resource_group` | string | Resource group name for the ML specific resources |
+| `sql_fuuly_qualified_domain_name` | string | The fully qualified domain name of the Azure SQL Server (e.g. myServerName.database.windows.net) |
+| `adf_id` | string | The ID of the Data Factory Schedule Trigger |
+| `private_endpoint_id` | string | The ID of the Private Endpoint |
+| `virtual_network_id` | string | The ID of the virtual network |
+
+## Customers/Persona
+
+- **Admin**: This persona represents an administrator of Azure. This persona does not implement the line of business applications but will help other teams deliver them.
+- **App Developer Team**: This persona is responsible for creating and maintaining the line of business applications
+
+### Credentials Management
+
+Managed Identities (MI, or MSI) will be used to manage access to credentials when possible. This table identifies MSI support for each resource leveraged by the template (full compatability breakdown [here](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/services-support-managed-identities)):
+
+| Azure Service | Supports MSI | Mitigation Strategy |
+|---|---|---|
+| App Service | Yes | N/A |
+| Key Vault | Yes | N/A |
+| Azure Functions | Yes | N/A |
+| Blob Storage | Yes | N/A |
+| Azure SQL | Yes | N/A |
+| Azure Data Factory | Yes (System Assigned only) | N/A |
+| Azure Container Registery | Yes | N/A|
+
+
+### Risks
+
+- Ingress / Egress for app services, functions and storage endpoints are not restricted to virtual network(s) by default. In summary, by default these services are exposed to the public internet.
+ - Mitigation strategy: App Service VNET access restriction can be enabled by setting the `vnet_subnet_id` setting. We still need to support VNET restrictions for the functions and ML workspace modules.
+
+## Security
+
+Here is an overview of the security for the deployment strategy and templates discussed above:
+
+- **Role Assignments**:
+
+The service principal running the deployment will have to be an owner in the target subscription and granted admin consent to the `Application.ReadWrite.OwnedBy` role in Microsoft Graph. This template creates a new service principal that will be available for application developer team(s) to administer the provisioned Azure resources. The following role assignments will be made to that service principal:
+ - Contributor access to App Service & App Insights in the first Resource Group
+ - Contributor access to the second and third Resource Groups. *If needed, we can lock this down further by granting role access to individual resources in this RG.*
+- **Service to Service communication**: For services owned by the application development team persona we will leverage Managed Service Identity (MSI) for authenticating to cloud services wherever possible. Where this is not possible, MSI will be used to pull credentials from Key Vault. This connection is done via MSI and therefore it keeps all secrets out of the codebase. This limits the attack vector of the infrastructure. [List of services that support MSI](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/services-support-managed-identities)
+
+### Role Assignments
+
+//Add the role assignments for the project in this section
diff --git a/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.drawio b/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.drawio
new file mode 100644
index 00000000..16158e8b
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.drawio
@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2020-05-20T23:55:25.572Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.0.1 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="je2vg6-IF5-uQiEGGDje" version="13.0.1" type="device"><diagram id="er0hbP_QVH_gZOtt2dKf" name="Page-1">7V1bd5u4Fv41ebSX7sBjmrSdWac5p2nOms6Zly5sFIcWgw/GufTXj2Rz37JDEkRMpn6wjZCF+PRpa9+ET+jZ8v5j6q9uLpJARicEBfcn9PyEEOxxoT50yUNe4gq2K1mkYZCXVQVX4U+ZF6K8dBMGct2omCVJlIWrZuE8iWM5zxplfpomd81q10nUvOrKX0hQcDX3I1j6NQyym12py1FV/psMFzfFlTHKzyz9onJesL7xg+SuVkTfn9CzNEmy3bfl/ZmMNHoFLrvffdhztuxYKuOsyw8u7y6//PhxGf3nr4e1L/+4isjXzSRv5daPNvkN553NHgoE0mQTB1I3gk7ou7ubMJNXK3+uz96pQVdlN9kyUkdYfS1vUh/Mk2U4z3+3ztLkhzxLoiTdNkvR9lWeKcClqgTeWdFNmWbyvlaU3+lHmSxllj6oKvnZieD5b3LiTXhxa3fVMGLC8sKb2hi6BWn9nDuLsvkKXvUlR/gJaBOA9s+XwX0dRlEN0+vrazKfH0R7kfpBqHBt/CrwhLA2EtxzmiPBikbqI4EpgyMhisnU+0hQm7y3hCNiTRwphjhyYYCRCWwJRgZg/CLXySZVKBH0USG4Up/4ALT4yRwPfOleGzku5q6cXfeENnPYlLfwJgLgXUr3Ot7UFmv5+FjL3BZriYCsJcyAInZssVYAGK9kehtuSfs58uMTIiLViXezVH1bZCUSI2MwBwwmDmQwdodksDNCBgsP4IjLojqSngFJj1tC0n0KkkdMUgguZSZwnSFp6o2Qpi6Y7gLCaJS0ji2OFkZcDcfTlVYJfo/X+vLrUVJWAKALEVoHGg3JV2zVjrNEWMPUR68vVzE00kbJUgbRJa8vWPEIDS8DlNg7AqJC42snXEtldoSspQZd6whYCw2vq81sPU/DVRYmcf82LpduwExou2RGt76aPtDmcCUz+WYYHhRraJ2d/tyk2jb7r4x9dbcjRFoQgDSnDkDaMwkQe0iPzxjD1G3xFUPVi1ICUSwDCv3DCC0xgxOMmJwKb8DNUK55hyTIsF4yPELrrUSoFA/e1DGYbyYgRVW1/4gFtOD+JXXH//A3irxE+EsNVjxb648TrsBAF1e/q/cTfj5COivytOS0w6fEoH8UtvUgjCbjM++w65W4laocQwZVjgiTcLDl+y2Qq6sX55+utKSW8U5Oo/HTWCmkpUwo0ecmL8WgNO5g/ClmrvTX60jen+r4vQJExkH+9Xwe+et1uMPPTzNYbCK5nhDyPsz+zE/o7//T5YqNu6Pz+1q184f8oHN0NRAzwcW2pwqnP4u29EHtOvqwutD2qLjS3iHP9Yj9mLLcClRoLGR2oGIx+jJoJDxACtWjiYaZWZSlMvKz8FY2umuiTH6Fz0modfdqxQMaMS404qKV3c3nP6y4B9qiCFiNFCnQUfUizZZ3aIGWt6wuoXgB0aFp/kJ5bUfpAI4NVQKXOpNRwq0pHAZTuwrOpWE8D1d+NEKRrFYegHaR81BH2zTn7ElkaGwDaJNNFoWxgqVIsNJ8Dfz1TUleDUI496NP/kxGn5N1uHWK0PNZkmXJslbhNAoX+kSW6BHx86O5AlWmQHZvl4Hl/UInlk39uzWd+kHwTTfcHkCHe2f6pkFyS5zEqpF3zal2NClK2PGadBDQI1DN0IYx69jiQweXgHE17nPZLRdR/LRFFI5SjxP90YWYeB0X4sLtciwLMQjDEGXytMR795WYA/+t57Rb27P6Klb5D7VqK11hvb/ngqFWv72DnWvXn+A8WaqaMbsu9KsLdAjU/5pSB9WoR6dUMfeOeEqJKRbVy3umpsuFN60100qsdNGLZ1rvmjD0xSlFIkvD2SbTGt2XJBpjoAq3xRwRr63KUeip60fOoCfImeOQF8ciBgiIZhK3Hzkw4bi9ljndTNynLrITgduGepFFuLdzAjNz56wutBQ6Sd+CqJkw3hTzRyBqyOOiZrGLe3W9/XIDjz8rWkAHYaHtXQWcuwAW6hiMJ1xoZP3j0sG/ucOlm0l5GPznQ/iKCPXuGKuvWM/HlO7BFGJX7GMz+NJNaRKOtTkI/WSf5EIt8C8TcQ08oXOkwBZbwbY4CxxlE6RWTYyx67r5OwwjuSYie1NKGRHcyd9teS1Z38qXHUXq4AAdidbUWuycKcWcukzs3mmzua76kmhrMAdbtRwdYKZo7i4VJAhvG6QR/9/o7aHv1MzJJrm/9FTViOR1Vp1tJ5SsV9s9LNuyndaDThXfFvFSMyGvpLper1cr3nZiT0KK7ogpNNHyuNbJmhcBb2/bKbwMg0Bfxiii9rhw21LKmkxqmdrYnSol1xMc0e27QSMzRVaHk0cdNLQeI61PD5SOUza1xIggU9TaKvyoQNr/U9tSx6ScWpM6VQLcuVwpVsl4/vBL8vQgeXhFnLq0Me1j30obWwLGmiJvMdxdbD6oMmJgwI0UvosGktyWGs8OhrvfzF7UMkG+2olKIYmLDb+DODFYh9D30TEYYbg9grz6ThQGo8ZFyv6HTTzfbZEYH2mxB9EmzqtvRuEdbM0jIy5IEicegZLXmAdKbKEIjbCCs1eXn0ZIV4IISIIzPCBk2IxP3kFNsJ3x2Yxfkb5TPvcO3ZHYLBPSDhW1nwbTOehEXKTMXsK84tVO4MS0YwC6L6OGQ/VpcH7hKUJug2OCsJ5J9ty8Yn3ZzzINFc7a8hkq1/jIpgDGU8FwyVq3vS8BCyUXvNrrmdlP2HVATrODwOYc23NijNv64IiAlYsy08plbaMI77Stj45QU8AeCM8b7OBhH2w1vi17GD4WCJv0rUF3NwloG1woZRZ9TdIf6y1YvdJ1kM3TuB3px8SdYug+GFazFdB+ANj+SuscQ7YmrtImyywt73kKAHObqjbGIJPaVu6ziw72rF0f40YOl52MLOcI9vv9062/MpentP7Y87gNWyof2TqQYut0MPaOTEkgCLV3QRsewUgQM+3ycW0Fb5x/xAMwtWe8hb3Bqhg26OCMzzbTue6treSYGHAUjonD1jIcHGievYGYg+ZsG2xkIu2Qxpk7voDDRMHRUuo4jNsM+zBM1xBxOP8wQpJOCBItcJlpURuUo+Rxjg6rag6maT5qbxWMfnxn6nEZZnxKMYLPcCgtmBaXOiuyQjWMSPl0CNxUa3nX7ap9qbXe+MSrXuzby5QwBMaHdX55B0K6WZLqP3oZn7DFHtQIDLJ2SNeXN77cLwNhJ4wYn7A2LGWhSXt69uVt0HTfI+yG5er4srxMT06jpkzFYZm6P8drvMLVkE9HHVM+nTXSeg/fN5dO8uXu28Xk/bfvy+//xrd2/5RrH2cN+wkBtvtNAua2o2FD2ltGFKFFUBD2IolDxdnXIewLcQZ/S8Sge6Av00sdVn9Pt1N3q3/5o+//Bg==</diagram></mxfile>
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.jpg b/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.jpg
new file mode 100644
index 00000000..9044721f
Binary files /dev/null and b/infra/templates/az-svc-data-integration-mlw/docs/design/images/deployment-topology.jpg differ
diff --git a/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.drawio b/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.drawio
new file mode 100644
index 00000000..569b953f
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.drawio
@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2020-07-16T15:13:20.066Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.0.3 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="j5yQWXBkMpMRw4GZDwEi" version="13.0.3" type="device" pages="2"><diagram id="XMOtGPsxv2L9FUZioUlo" name="Page-1">7V1bc9u2Ev41nul5kIcA74+O06aZ056Txr0+ZWgJlnlCkRqKsuP8+gNKBC1gYZGECJCU3c60EiStrP12F3vD4sK+Xn37kEfr+1+zBUkusLX4dmG/v8AYWUFA/1euPFUroe3vV5Z5vKjWnhdu4u+EfbRa3cYLsuHeWGRZUsRrfnGepSmZF9xalOfZI/+2uyzhv3UdLQlYuJlHCVz9K14U9/vVwLWe138m8fKefTOyqldWEXtztbC5jxbZ48GS/eOFfZ1nWbF/tPp2TZKSe4wv+8/99MKr9R+Wk7Ro84GrzW8ffvz86aP/z+32arb4azGbuzNvT+UhSrbVD46+zzYP89kiKqJZnBZkmUdFnKWzVfJY/Y7iiTEnz7bpgpT00YX97vE+LsjNOpqXrz5SeaBr98UqqV7eFHn2tWYi/fnv7uIkuc6SLN9Rs+/c8t/6nQeveLt/yk9kaXGwvv+HrkNeVOx5IHlBvh0sVbz5QLIVKfIn+pbq1aCCqRJUjwng4zPqyKvW7g8QZ2tRJWjLmvIzFvRBBUcHaHwADeA/SRdXpYzTZ2mWEp7f5Ftc/H3w+B/62Lp0q2fvy59tsSdP1ZMX+bjJtvmcNAsSWXDqBLl9wE5Xwk22lpOEit0Dr4QyFlff8CmjsvoMpuPwaPpIQGn/e6pPHSqNQAiIhSMQKqJ8SQpAaId4/bPVhSBoFoIl1cJ1ewWoTWV0yyhYcqgYLxHHgQDqRW3iDpFEgS7FQGEL1XiBK0eZfDKrZpiXFgxZ5chY5WpjFWDU7/eELvxOVmuqYHRjxV60Kg11ertZ7wyA1WoJPE/ob3l3m9NHy0K6cp3dRkmpWdRJ2CYnfHPjN1193+blj/xM9lre41e9uP9ZKvtflMTLlD7O96Igsb4N8t+8uTG3C7WURBRqkkTcrLIj9iZOxqF61efMgw1NaSABBeuyDrZpTBYuCRaODJMA39o7TEzx3g2H5b1jnPcRCe7mUn2YB+T2TivvPY73IbRGRnnvAt7/QpbUl+4bAX38RHZL8cWaWBg2i+/R4ASyppKJkQQQOOQ9OeT2FUEgkZDmCIJ9XUuo5km02cTziaEFeGyrgWV7DYR0gwUd9jMEC3M8psH5pasKVyMp3YDBvSRar79sSP4Q031ihO5U58SY7fEJAOSHjMmHuTFLIjG2trgW5i0rln+hcW16JnznhTuEW75hpnfLSE7UOAkbQOj0tJOIhDQbJgx3krttOi9T+WepHdj1B1YPu0Wudurq4Yh7gap6eFhNPSj3oqeDt63LN2yO/MGh/A9+BnpPsVfdc2Bi5Wq9pgs3zC2wHinhmyJaxumyXE+yQrdWqoT/nbUSwOr7El8hlKmlgH9/iZYWmRbdJRTX5dkyQ5K9XFZcRJavy1q1SP4ZKC8ehayx6IjGVXWcCTCjoGsYyj5omw077RY60jGdcxTYofFibPa67l7sg1bPuxWj6xjYnWwYOx0Du6WfMgm8LcW0kC/otUhHt3pqCbwmgZjXE2IinZ7cS1v+NXr1F+ben/3KT7vsB/2ryor2vo0NW3+swdJ/t6N0OI97cO1LfUL3D6yzylxQbZ1cDgaIifHAx+dOu9eDC3K8VsDoiw1g9hZw33RsgC3IFMOxgdPNQ1B3B91xbDbMfWfNyqzXz+u497zwOc3egaMl7zRqwFQBkpshw3ixH3+4G1Qdaz8ZytD2twUwA9o+QxRyvHfbbc3akrYedK1LF2lnvLFFd+kknle7svWerJPsaUUqj5ADaHMfrcuHm4KUaKxJHtM/kOTV0qfn503o3cXfCDsHUT7noW9ue7u7w3Mptgvv1nN7yr77gcPv5DiAmxaS+Vi69iyvdae0yMRu5wc67+++y7NqJun6k7Vi2oGni1Wnpv6UO50YSEPvH6zYx+8DthgAtG7xEfYTsw0+LIn1euHsGT9ARzeALaKQVwEg68k9ET+RjG74To2WJg+fHAYUKqqj00BHN54telt1JwXAsbNQkhRgfXRmmn+67TKnRJkMgJGINX9uACs2PAiAimR6ykcj6R/7oqIde7ee5DWCylU37uVxOo/XySi7Jo9bivaHjDHHcUnQZLRlyR9B/lPUDAdLeiMCZNLUdYuPTjF1/rgSamaNW282BYb+ZQ/2KkvjIst3HVYTMSh+53QaUwzWIzywQQm6nQuaZA8kZl09LNxAvvr5hUZSmnWH+daC7sTpZse3ESqOQqewwGTWvTGYlrCe1HGM9hiJWjlCQ7fjYlW1EkuqElK61Qqfvx1swWXVbvABAIM+368H/oMVlaeDrauEWh0DDsUgvdoOa7MYqhsj7JZieC1mUXQRQnWziJpI6daybpnuiZrFRi63N4uDAwZz22yqDl398JwaODNb6AbQFkrr2fqMYesZV6br2cI8MNeCQact4RQW5x70x6puDVA9FmAYSCMxPSz8Z3VM+9ILQoRC5Fth4DhC60Xr6hpPdSb2Jeg2QaeOGjkXdJHLjH+/+NpWN7ra55XAlMQrBVxoHBIVbyIKjKxT0x0QUIpj/vT3zr23ApctVP4+qt/x7PLvnj0dPjtoyetyNpHJyJ5pR382HpU0iVKA3UvPxoiGfS7VeKQoW44go9i0bGEgW3zT5j5wj9O7PKJitJ0X+07b/96WZcfoNk7i4glI46vr5uQ3AMeR1BGMdnPWZxvecgFc5QFzOHniMdTWW77YAd/yPGt/avsKZmU0Mlk1O2oeLeiC80b2475cBNviJ5kLQPxO6TL/Zai8KGozeEn7MTWH54rnSapo0mNq2uJ+JCn8d9skIL+OAzASy1KX2ZmEds0vvvA57cMNTw3lp4YXa/Pq2gjLAnHUCp+uUyZ8JKCro88Pd4viT2miGhvcYt+A4rFEkYxu7ZRMjHstiAWoF8REMnqGTARG1BfG1QdDJvadulGyA4jig63P766ugbSMwPtrcKhUJxe0m2mm71oPDNsVPmf7I6tUKZdpdUjV+qHE5V+vCBjmgQ6HzChOj/Dz9gMEfXUs7Xg7YjlPbKk26A4wCEayuSDF6r84idaws25Dd+AreXqItknxZU5FVLdR6e+qkFolVY9pwPi/Dj0NHUiAu/EARoVPEgcuhkZFdhEY++M1sKWHgxrPlSShitRQQ2orfOzXN9eLWP5oLGZLsZzs82Ji3GzBJkJmtmbrLInnsNozWsNVq317w8WX46CGIlboM2S4WpyN1m+4+GOOQQgnhcu9IX1sMTfQsYZgaLPCcpfC1qoaeQdHyWi3MjDzzKzMhMwLU07VO3LZJWaH9kXqAehTpH6bw6ZQVhSPXYTiGEDVUzK+4TkLyOkWE76Bd9h3I2xqgJB28Ppt6uJccXSqKy4Hvb0TPhbxCNiEgc66bQnBmtipo108WkSsZ67bAe5JtwEh7eBpmfUyMrSEfoaAjVHt3J2DGghpRwseqvo3KRn5594dteqLrff/vSjnTs9zUmx+uLn55WJXznmIirJ4QD+3GWW14OQmnoCJMBcLYyhe2oaAozZTlQeaUMlvFl4I59LITvTYzMhr4NWp7Z/ql/E644qXEY8NttBlWF50Gfoh9uvruLoaLYFqfXeZMZs12JGt0eF7qQdgrxtd7YAPdoprNIDXuqZ41ALorOGjFpK55adCONwxnloqmkNCd1yOqyAGgXVpI9sOfSewbcf1VCcRibmgS5/GcIHL/mtY1GBygbqs2/ztrA44q+OzYG24szqsEHzO4aJY/UbIddSnfjXT0q5hsHS+KbI8WpIv0XxOBX+UdQyF2V8iq1nFYLhbtN1u2ZVzSI05UOIV73VzxWu1ASXtmtPDDJvpIdjMd+Wbn80jCBMxN3vbRxevzFi/QZJg1PoNfBkzcmFi55n311laRHFK8jM5SAgkPRj4vrv6gN4r2nvEsgzCqudwAaV6KzdluTwYF72GvQciqLr3oECkZHrQJGJjTs87WuIPIITeCcFSIyntiMFYaZXMHrP862a3BZ1HoCSwWXIxn+E4qeO9VhNVFL7Uj6zgBE1ppqVdVWBwNM82q2yzuD0TNRF5LLlb2rSemLh/alxq47OEdL2POwJ7W/cRi5QskZJ2lTFR6h4ZfK4nwGf7l5ZVFydV74GDYnGcrnZoTVS5Rw4t0KcJaWb/NWsjBc/Wza5s7xqL7IiIIzvgipuKZsHmyTq2UNw0K1U+TMHww7QWZE2liqTzmLzdMy0Ekgix5M1w5c7gzd9CFu7JqociIe13hb25WxQ9pMXdaqCrHdo3dysUSajrpeGWn443x03P2QrH7WyFLN12oneFA967MtxYLLnP7tfyjMMF9hLKwHe3OX20LB99FEZCvzlanDjYkryWWT/r5FvyplfpcpqLHe0bZYB+m04G93EJ3vQwbMF41fufhsAQ1r6MFb4GadIInfrkxGBdGm0uNNI+NSXk+YJsSUXQs0LIFyQO4+iPMX63LeGkuSkMhKHNCstaBapWRLxfHRDSnvGC+0BKitKM6LYg/c1KqVVSdVaK5HJNw3VCf7gzd2PRpfqAvcPbNqR49hsQMhzt+MOdqhsbpKxYU0MacqmoQHkWgygqx+lqT5P0fwjvXBBXzGIChE1nLfsNcwc+VsmEpDkfxmasjkWcBDFgbZpdxWmGhZTszEP48uAkpTg0Qbt8Qe/rP3vvS3aR+bTTXn35bk7I2xYMB2KYLjp2G3WrlgybhN1HgmMtjnJvfSumMKhWpKNdL3tobz0bUJvAaF2MsAcGFfa//kmtLQB2BJmyvmylbfMbJxtabCB7hv/3288ojP9Y3d1/8MiVb1vBbzPJ6IDtbToeDCSpubawtMbAhxOODWcwuwWmE+3dFwIQHLrqvfvNtDTbLiy5d3vODmTOcrKMqQOYw/kbI0jZKbTxi+wOYHJb6uFpS8/hnm/JnoIGOT6QesXZSK7vNFDSrj0wzOps8Tog2Bhf12ckR4K1OG9THWvPHhxrGIzVR9fp8mdTtnKYIQL17P3Gse+63AtsdZvBcRa2Ep7uU/Y2XJbcOkJLuw5166I+m+aQZs637w4ZAYqwm/rq+66dzrrenfukD96/Ow87KDTjIMvzDXaKSGPdbj5HezMIBucDzg2nQEJbiuOouoyAkqUtWSQFr4fLy84CP2Xz10RJM366jgGNFzxfjLJs1aH5gBLWNzVfCp6uU0AjBs8NRfDCfk4BAaE4SlczsLqanCYELHb6ghJQ0gzehM4ANcsDn6SR1ype6MEdyJxj1rF/etODbxtqejjCV06QPq72MwbhQaDPZJ1t4iKT5Gum1w3RUF1qOgQkWhLXYOODHMneOr7Ha8BBq72LLOVCkt9MS7fywah0ERXR7C6aS5VsoALSSZoC2OyEcLSntv5uOdv77RCahOcjbmB1THiy5+MEZpMBqFs30FmgJ6YCPJFGB/SE225KWgMGIajf+sQk0BSjECdQR7OBkm70+s/sjDwM8UdlxV3V+BWGIY7vDhmGwCzTe+oJSaOQj2lBlnk1A+otDBF8WFhnMRyG9JtWmkJU4sBIQjE74Iq3fwBKuvWwh27E6QHYzPb2pZaBAcQwn1MZ0p/MBJU9Nfc2VZhFxyeEh4bN1pdh9qXk+2Yy3dQdGe5Kbocyy3CYOzn3fQZUhF1xZpRyRd+4mXor6e8BVDxsDUr6gJJuAGHc/ilekyRORzPspmeDFwy9w/Qba09SXzzV62yaNU+3vnRrozhXg+ex5v+TDR6gpBvAbpOmWo+V6mKUsHDyvs4HHaCJpaWUHuZMybnS2+m1o0J8BBHjol1bEDVBDkRDZFSKbRgefiVPD9E2Kb5syDzXH670VHTsJBGM9ZhnPdCderqRkTKkDUPGY7pzFhu6cLSpzl92H0XjHSekrEf0aZ5lxeHbqZTf/5otSPmO/wM=</diagram><diagram id="6HlrVojdOWclmM4p_VT2" name="Page-2">lZHLDoIwEEW/pkuTQn2wR8UYjYoxrhs6QpPCkFIF/XoxFLFxo6tOz9x5ExbmTaR5mW1RgCI+FQ1hc+L7Hg2C9nmRe0cCNutAqqWwogEc5QP6SEuvUkDlCA2iMrJ0YYJFAYlxGNcaa1d2QeVWLXkKX+CYcPVNz1KYzE4xoQNfgUyzvrJHrSfnvdiCKuMC6w/EFoSFGtF0Vt6EoF7L6/cSj6MabjHS6elQbdL9Y70zoy7Z8p+Q9wgaCvNr6tYYWms/zoHZ4gk=</diagram></mxfile>
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.jpg b/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.jpg
new file mode 100644
index 00000000..9ae4fa7d
Binary files /dev/null and b/infra/templates/az-svc-data-integration-mlw/docs/design/images/template-design.jpg differ
diff --git a/infra/templates/az-svc-data-integration-mlw/keyvault.tf b/infra/templates/az-svc-data-integration-mlw/keyvault.tf
new file mode 100644
index 00000000..f22545fa
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/keyvault.tf
@@ -0,0 +1,62 @@
+locals {
+ secrets_map = {
+ # App Insights Secrets
+ appinsights-key = module.app_insights.app_insights_instrumentation_key
+ mlw-appinsights-key = module.mlw_app_insights.app_insights_instrumentation_key
+ # Storage Account Secrets
+ sys-storage-account-key = module.sys_storage_account.primary_access_key
+ app-storage-account-key = module.app_storage_account.primary_access_key
+ app-sp-tenant-id = data.azurerm_client_config.current.tenant_id
+ # AAD Application Secrets
+ aad-client-id = module.ad_application.azuread_app_ids[0]
+ # Service Principal Secrets
+ app-sp-username = module.app_management_service_principal.service_principal_application_id
+ app-sp-password = module.app_management_service_principal.service_principal_password
+ # Cosmos Cluster Secrets
+ cosmos-endpoint = module.cosmosdb.properties.cosmosdb.endpoint
+ cosmos-primary-key = module.cosmosdb.properties.cosmosdb.primary_master_key
+ cosmos-connection = module.cosmosdb.properties.cosmosdb.connection_strings[0]
+ #Azure function App
+ function-app-key = "This will be updated in Azure DevOps pipline"
+ }
+
+ output_secret_map = {
+ for secret in module.keyvault_secrets.keyvault_secret_attributes :
+ secret.name => secret.id
+ }
+ app_setting_kv_format = "@Microsoft.KeyVault(SecretUri=%s)"
+}
+
+module "keyvault" {
+ source = "../../modules/providers/azure/keyvault"
+ keyvault_name = local.kv_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ subnet_id_whitelist = module.network.subnet_ids
+}
+
+module "keyvault_secrets" {
+ source = "../../modules/providers/azure/keyvault-secret"
+ keyvault_id = module.keyvault.keyvault_id
+ secrets = local.secrets_map
+}
+
+
+module "function_app_keyvault_access_policy" {
+ source = "../../modules/providers/azure/keyvault-policy"
+ vault_id = module.keyvault.keyvault_id
+ tenant_id = module.function_app.function_app_identity_tenant_id
+ object_ids = module.function_app.function_app_identity_object_ids
+ key_permissions = ["get", "list"]
+ secret_permissions = ["get", "list"]
+ certificate_permissions = ["get", "list"]
+}
+
+module "adf_keyvault_access_policy" {
+ source = "../../modules/providers/azure/keyvault-policy"
+ vault_id = module.keyvault.keyvault_id
+ tenant_id = module.data-factory.adf_identity_tenant_id
+ object_ids = [module.data-factory.adf_identity_principal_id]
+ key_permissions = ["get", "list"]
+ secret_permissions = ["get", "list"]
+ certificate_permissions = ["get", "list"]
+}
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/output.tf b/infra/templates/az-svc-data-integration-mlw/output.tf
new file mode 100644
index 00000000..6fc2ead5
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/output.tf
@@ -0,0 +1,114 @@
+# output "fqdns" {
+# value = [
+# for uri in concat(module.app_service.app_service_uris) :
+# "http://${uri}"
+# ]
+# }
+
+# output "webapp_names" {
+# value = concat(module.app_service.app_service_names)
+# }
+
+output "app_insights_id" {
+ value = module.app_insights.app_insights_app_id
+}
+
+output "service_plan_name" {
+ value = module.func_app_service_plan.service_plan_name
+}
+
+output "service_plan_id" {
+ value = module.func_app_service_plan.app_service_plan_id
+}
+
+output "app_dev_resource_group" {
+ value = azurerm_resource_group.app_rg.name
+}
+
+output "keyvault_name" {
+ value = module.keyvault.keyvault_name
+}
+
+output "keyvault_uri" {
+ value = module.keyvault.keyvault_uri
+}
+
+output "keyvault_secret_attributes" {
+ description = "The properties of all provisioned keyvault secrets"
+ value = {
+ for secret in module.keyvault_secrets.keyvault_secret_attributes :
+ secret.name => {
+ id = secret.id
+ value = secret.value
+ version = secret.version
+ }
+ }
+ sensitive = true
+}
+
+output "acr_name" {
+ value = module.container_registry.container_registry_name
+}
+
+output "sys_storage_account" {
+ description = "The name of the ml storage account."
+ value = module.sys_storage_account.name
+}
+
+output "sys_storage_account_id" {
+ description = "The resource identifier of the ml storage account."
+ value = module.sys_storage_account.id
+}
+
+output "sys_storage_account_containers" {
+ description = "Map of ml storage account containers."
+ value = module.sys_storage_account.containers
+}
+
+output "app_storage_account_name" {
+ description = "The name of the dataprep storage account."
+ value = module.app_storage_account.name
+}
+
+output "app_storage_account_id" {
+ description = "The resource identifier of the dataprep storage account."
+ value = module.app_storage_account.id
+}
+
+output "app_storage_account_containers" {
+ description = "Map of dataprep storage account containers."
+ value = module.app_storage_account.containers
+}
+
+output "mlw_id" {
+ description = "The azure machine learning workspace ID."
+ value = module.ml_workspace.id
+}
+
+output "mlw_dev_resource_group" {
+ value = azurerm_resource_group.mlw_rg.name
+}
+
+output "data_factory_name" {
+ value = module.data-factory.data_factory_name
+}
+
+output "azure_functionapp_name" {
+ value = module.function_app.azure_functionapp_name
+}
+
+# This output is required for proper integration testing.
+output "cosmosdb_resource_group_name" {
+ value = module.cosmosdb.resource_group_name
+}
+
+# This output is required for proper integration testing.
+output "cosmosdb_account_name" {
+ value = module.cosmosdb.account_name
+}
+
+output "properties" {
+ description = "Properties of the deployed CosmosDB account."
+ value = module.cosmosdb.properties
+ sensitive = true
+}
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/security.tf b/infra/templates/az-svc-data-integration-mlw/security.tf
new file mode 100644
index 00000000..f97d45dc
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/security.tf
@@ -0,0 +1,93 @@
+###
+# This next block of Terraform configures the Service Principal
+# that will be used by the application teams to deploy service
+# code, configure and manage KeyVault secrets and manage App
+# Service plans, among other things.
+###
+
+locals {
+ rbac_contributor_scopes = concat(
+ # App services and the associated slots -- enables management of deployments, etc...
+ # Note: RBAC for slots is inherited and does not need to be configured separately
+ # module.app_service.app_service_ids,
+
+ # The ML workspace Id
+ [module.ml_workspace.id],
+
+ #The Function App Id
+ module.function_app.function_app_ids,
+ )
+
+ contributor_role_name = "Contributor"
+ storage_contributor_role_name = "Storage Blob Data Contributor"
+
+}
+
+module "ad_application" {
+ source = "../../modules/providers/azure/ad-application"
+ resource_access_type = "Scope"
+ ad_app_config = [
+ {
+ app_name = local.ad_app_name
+ reply_urls = []
+ }
+ ]
+ resource_app_id = local.graph_id
+ resource_role_id = local.graph_role_id
+}
+
+module "app_management_service_principal" {
+ source = "../../modules/providers/azure/service-principal"
+ create_for_rbac = true
+ display_name = local.ad_app_management_name
+ role_name = "Contributor"
+ role_scopes = local.rbac_contributor_scopes
+}
+
+# resource "azurerm_role_assignment" "aks_acr_pull" {
+# role_definition_name = "AcrPull"
+# principal_id = module.app_management_service_principal.service_principal_object_id
+# scope = module.container_registry.container_registry_id
+# }
+
+# Create a Role Assignment with Storage Blob Data Contributor role for Data Factory to access storage account
+resource "azurerm_role_assignment" "adf_app_storage_role_ra" {
+ role_definition_name = local.storage_contributor_role_name
+ principal_id = module.data-factory.adf_identity_principal_id
+ scope = module.app_storage_account.id
+}
+
+# Create a Role Assignment with Storage Blob Data Contributor role for Machine Learning to access storage account
+resource "azurerm_role_assignment" "mlw_sys_storage_ra" {
+ role_definition_name = local.storage_contributor_role_name
+ principal_id = module.ml_workspace.mlw_identity_principal_id
+ scope = module.ml_workspace.id
+}
+
+# Create a Role Assignment with Storage Blob Data Contributor role for Machine Learning to access storage account
+resource "azurerm_role_assignment" "mlw_app_storage_ra" {
+ role_definition_name = local.storage_contributor_role_name
+ principal_id = module.ml_workspace.mlw_identity_principal_id
+ scope = module.app_storage_account.id
+}
+
+# Create a Role Assignment with Contributor role for Machine Learning to access Container Registry
+resource "azurerm_role_assignment" "mlw_acr_ra" {
+ scope = module.container_registry.container_registry_id
+ role_definition_name = local.contributor_role_name
+ principal_id = module.ml_workspace.mlw_identity_principal_id
+}
+
+# Create a Role Assignment with Contributor role for Machine Learning to access Keyvault
+resource "azurerm_role_assignment" "mlw_kv_ra" {
+ scope = module.keyvault.keyvault_id
+ role_definition_name = local.contributor_role_name
+ principal_id = module.ml_workspace.mlw_identity_principal_id
+}
+
+# Create a Role Assignment with Contributor role for Machine Learning to access Application Insights
+resource "azurerm_role_assignment" "mlw_ai_ra" {
+ scope = module.app_insights.id
+ role_definition_name = local.contributor_role_name
+ principal_id = module.ml_workspace.mlw_identity_principal_id
+}
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/storage.tf b/infra/templates/az-svc-data-integration-mlw/storage.tf
new file mode 100644
index 00000000..f53dcae2
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/storage.tf
@@ -0,0 +1,63 @@
+locals {
+ collection_throughput = 40000
+ db_throughput = 1000
+ cosmos_databases = [{
+ name = local.cosmos_db_name
+ throughput = local.db_throughput
+ }]
+
+ cosmos_sql_collections = [{
+ name = "scoring-data"
+ database_name = local.cosmos_db_name
+ partition_key_path = "/APPT_ID"
+ throughput = local.collection_throughput
+ },
+ {
+ name = "scheduling-data"
+ database_name = local.cosmos_db_name
+ partition_key_path = "/id"
+ throughput = local.collection_throughput
+ }]
+}
+
+module "sys_storage_account" {
+ source = "../../modules/providers/azure/storage-account"
+ name = local.sys_storage_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ container_names = var.sys_storage_containers
+}
+
+module "app_storage_account" {
+ source = "../../modules/providers/azure/storage-account"
+ name = local.app_storage_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ container_names = var.app_storage_containers
+}
+
+module "cosmosdb" {
+ source = "../../modules/providers/azure/cosmosdb"
+ account_name = local.cosmos_account_name
+ resource_group_name = azurerm_resource_group.app_rg.name
+ vnet_subnet_id = module.network.subnet_ids[0]
+ primary_replica_location = var.primary_replica_location
+ databases = local.cosmos_databases
+ sql_collections = local.cosmos_sql_collections
+}
+
+# //storage for ADF data prep
+# module "storage_account" "adf_dataprep_storage" {
+# source = "../../modules/providers/azure/storage-account"
+
+# name = local.storage_name
+# resource_group_name = azurerm_resource_group.app_rg.name
+# container_names = var.storage_containers
+# }
+
+# //storage for ML ops workspace
+# module "storage_account" "ml_workspace_storage" {
+# source = "../../modules/providers/azure/storage-account"
+
+# name = local.storage_name
+# resource_group_name = azurerm_resource_group.app_rg.name
+# container_names = var.storage_containers
+# }
diff --git a/infra/templates/az-svc-data-integration-mlw/terraform.tfvars b/infra/templates/az-svc-data-integration-mlw/terraform.tfvars
new file mode 100644
index 00000000..f5ea4a78
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/terraform.tfvars
@@ -0,0 +1,58 @@
+# Note: These values will impact the names of resources. If your deployment
+# fails due to a resource name collision, consider using different values for
+# the `name` variable or increasing the value for `randomization_level`.
+
+resource_group_location = "southcentralus"
+prefix = "smpl"
+randomization_level = 8
+
+
+app_services = [{
+ app_name = "smpl"
+ app_command_line = null
+ image = null
+ linux_fx_version = ""
+ app_settings = {
+ service_domain_name = "contoso.com"
+ }
+}]
+
+# Note: this is configured as such only to test IP Whitelists. This is a well
+# known DNS address
+
+
+sys_storage_containers = []
+
+app_storage_containers = [
+ "sample",
+ "acidatadriftdata",
+ "configuration",
+ "source",
+]
+
+subnet_service_endpoints = [
+ "Microsoft.Web",
+ "Microsoft.ContainerRegistry",
+ "Microsoft.Storage",
+ "Microsoft.KeyVault",
+ "Microsoft.AzureActiveDirectory",
+ "Microsoft.AzureCosmosDB",
+]
+
+func_subnet_service_endpoints = [
+ "Microsoft.Web",
+ "Microsoft.ContainerRegistry",
+ "Microsoft.Storage",
+ "Microsoft.KeyVault",
+ "Microsoft.AzureActiveDirectory"
+]
+
+fn_app_config = {
+ "mlops-orchestrator" = {
+ image = "",
+ zip = "",
+ hash = "",
+ }
+}
+
+primary_replica_location = "eastus2"
\ No newline at end of file
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/integration/appservices.go b/infra/templates/az-svc-data-integration-mlw/tests/integration/appservices.go
new file mode 100644
index 00000000..c6d6614d
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/integration/appservices.go
@@ -0,0 +1,52 @@
+package test
+
+import (
+ "crypto/tls"
+ "strings"
+ "testing"
+
+ httpClient "github.com/gruntwork-io/terratest/modules/http-helper"
+ "github.com/microsoft/cobalt/test-harness/terratest-extensions/modules/azure"
+ "github.com/microsoft/terratest-abstraction/integration"
+ "github.com/stretchr/testify/require"
+)
+
+// Verifies that the provisioned webapp is properly configured.
+func verifyAppServiceConfig(goTest *testing.T, output integration.TerraformOutput) {
+ resourceGroup := output["app_dev_resource_group"].(string)
+
+ for _, appName := range output["webapp_names"].([]interface{}) {
+ appConfig := azure.WebAppSiteConfiguration(goTest, SubscriptionID, resourceGroup, appName.(string))
+ linuxFxVersion := strings.Trim(*appConfig.LinuxFxVersion, "{}")
+ expectedLinuxFxVersion := ""
+ require.Equal(goTest, expectedLinuxFxVersion, linuxFxVersion)
+ }
+}
+
+// Verifies that the provisioned webapp is properly configured.
+func verifyAppServiceEndpointStatusCode(goTest *testing.T, output integration.TerraformOutput) {
+ for _, fqdn := range output["app_service_fqdns"].([]interface{}) {
+ require.True(goTest, httpGetRespondsWithCode(goTest, fqdn.(string), 401))
+ }
+}
+
+// Validates that the service responds with provided HTTP Code.
+func httpGetRespondsWithCode(goTest *testing.T, url string, code int) bool {
+
+ tlsConfig := tls.Config{}
+ statusCode, _, err := httpClient.HttpGetE(goTest, url, &tlsConfig)
+ if err != nil {
+ goTest.Fatal(err)
+ }
+
+ return statusCode == code
+}
+
+func arrayContains(arr []interface{}, str string) bool {
+ for _, a := range arr {
+ if a == str {
+ return true
+ }
+ }
+ return false
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/integration/integration_test.go b/infra/templates/az-svc-data-integration-mlw/tests/integration/integration_test.go
new file mode 100644
index 00000000..55cbec60
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/integration/integration_test.go
@@ -0,0 +1,51 @@
+package test
+
+import (
+ "os"
+ "testing"
+
+ "github.com/gruntwork-io/terratest/modules/terraform"
+ cosmosdbIntegTests "github.com/microsoft/cobalt/infra/modules/providers/azure/cosmosdb/tests/integration"
+ datafactIntegTests "github.com/microsoft/cobalt/infra/modules/providers/azure/data-factory/tests/integration"
+ funcAppIntegTests "github.com/microsoft/cobalt/infra/modules/providers/azure/function-app/tests/integration"
+ mlWorkspaceTests "github.com/microsoft/cobalt/infra/modules/providers/azure/ml-workspace/tests/integration"
+ storageIntegTests "github.com/microsoft/cobalt/infra/modules/providers/azure/storage-account/tests/integration"
+ "github.com/microsoft/terratest-abstraction/integration"
+)
+
+// SubscriptionID -
+var SubscriptionID = os.Getenv("ARM_SUBSCRIPTION_ID")
+
+var tfOptions = &terraform.Options{
+ TerraformDir: "../../",
+ BackendConfig: map[string]interface{}{
+ "storage_account_name": os.Getenv("TF_VAR_remote_state_account"),
+ "container_name": os.Getenv("TF_VAR_remote_state_container"),
+ },
+}
+
+// Runs a suite of test assertions to validate that a provisioned set of app services
+// are fully funtional.
+func TestAppSvcPlanSingleRegion(t *testing.T) {
+ testFixture := integration.IntegrationTestFixture{
+ GoTest: t,
+ TfOptions: tfOptions,
+ ExpectedTfOutputCount: 21,
+ TfOutputAssertions: []integration.TerraformOutputValidation{
+ //verifyAppServiceConfig,
+ /* Now that we configured the services to run as Java containers via linux_fx_version,
+ we'll have to temporarily comment out the call to verifyAppServiceEndpointStatusCode...
+ The service(s) will be unresponsive until our Azure Pipeline deploys a jar
+ to the target app service. We'll remove the comment once our service CI/CD pipelines are in place.
+ verifyAppServiceEndpointStatusCode,
+ */
+ storageIntegTests.InspectStorageAccount("sys_storage_account", "sys_storage_account_containers", "app_dev_resource_group"),
+ storageIntegTests.InspectStorageAccount("app_storage_account_name", "app_storage_account_containers", "app_dev_resource_group"),
+ mlWorkspaceTests.VerifyCreatedMLworkspace(SubscriptionID, "mlw_dev_resource_group", "mlw_id"),
+ datafactIntegTests.VerifyCreatedDataFactory(SubscriptionID, "app_dev_resource_group", "data_factory_name"),
+ funcAppIntegTests.VerifyCreatedFunctionApp(SubscriptionID, "azure_functionapp_name"),
+ cosmosdbIntegTests.VerifyProvisionedCosmosDBAccount(SubscriptionID, "cosmosdb_resource_group_name", "cosmosdb_account_name"),
+ },
+ }
+ integration.RunIntegrationTests(&testFixture)
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/integration/service_principal.go b/infra/templates/az-svc-data-integration-mlw/tests/integration/service_principal.go
new file mode 100644
index 00000000..11f543f2
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/integration/service_principal.go
@@ -0,0 +1,55 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/cobalt/test-harness/terratest-extensions/modules/azure"
+ "github.com/microsoft/terratest-abstraction/integration"
+ "github.com/stretchr/testify/require"
+)
+
+// Verifies that the correct roles are assigned to the provisioned service principal
+func verifyServicePrincipalRoleAssignments(t *testing.T, output integration.TerraformOutput) {
+ actual := getActualRoleAssignmentsMap(t, output)
+ expected := getExpectedRoleAssignmentsMap(output)
+
+ // Note: there may be other role assignments added outside of this template. A good
+ // example of this is the role assignment that enables the services to create
+ // signed URLs for data that exists in storage accounts not managed by this
+ // template.
+ for k, v := range expected {
+ require.Equal(t, v, actual[k], "Expected role assignment is incorrect!")
+ }
+}
+
+// Queries Azure for the role assignments of the provisioned SP and transforms them into
+// a simple-to-consume map type. The returned map will have a key equal to a scope and
+// a value equal to the role name
+func getActualRoleAssignmentsMap(t *testing.T, output integration.TerraformOutput) map[string]string {
+ objectID := output["contributor_service_principal_id"].(string)
+ assignments := azure.ListRoleAssignments(t, SubscriptionID, objectID)
+ assignmentsMap := map[string]string{}
+
+ for _, assignment := range *assignments {
+ scope := assignment.Properties.Scope
+ roleID := assignment.Properties.RoleDefinitionID
+ roleName := azure.RoleName(t, SubscriptionID, *roleID)
+ assignmentsMap[*scope] = roleName
+ }
+
+ return assignmentsMap
+}
+
+// Constructs the expected role assignments based off the Terraform output
+func getExpectedRoleAssignmentsMap(output integration.TerraformOutput) map[string]string {
+ expectedAssignments := map[string]string{}
+ expectedAssignments[output["service_plan_id"].(string)] = "Contributor"
+ expectedAssignments[output["container_registry_id"].(string)] = "Contributor"
+ expectedAssignments[output["storage_account_id"].(string)] = "Storage Blob Data Contributor"
+
+ for _, appID := range output["app_service_ids"].([]interface{}) {
+ expectedAssignments[appID.(string)] = "Contributor"
+ }
+
+ return expectedAssignments
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/appservice.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/appservice.go
new file mode 100644
index 00000000..5db9b6fc
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/appservice.go
@@ -0,0 +1,36 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendAppServiceTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedAppServicePlan := asMap(t, `{
+ "kind": "Linux",
+ "reserved": true,
+ "sku": [{ "capacity": 1, "size": "P3v2", "tier": "PremiumV2" }]
+ }`)
+ description["module.service_plan.azurerm_app_service_plan.svcplan"] = expectedAppServicePlan
+
+ expectedAppService := asMap(t, `{
+ "identity": [{ "type": "SystemAssigned" }],
+ "enabled": true,
+ "site_config": [{
+ "always_on": true
+ }]
+ }`)
+ description["module.app_service.azurerm_app_service.appsvc[0]"] = expectedAppService
+
+ expectedAppServiceSlot := asMap(t, `{
+ "name": "staging",
+ "identity": [{ "type": "SystemAssigned" }],
+ "enabled": true,
+ "site_config": [{
+ "always_on": true
+ }]
+ }`)
+ description["module.app_service.azurerm_app_service_slot.appsvc_staging_slot[0]"] = expectedAppServiceSlot
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/autoscale.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/autoscale.go
new file mode 100644
index 00000000..57c423aa
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/autoscale.go
@@ -0,0 +1,41 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendAutoScaleTests(t *testing.T, description unit.ResourceDescription) {
+ v := asMap(t, `{
+ "enabled": true,
+ "notification": [{
+ "email": [{
+ "send_to_subscription_administrator": true,
+ "send_to_subscription_co_administrator": true
+ }]
+ }],
+ "profile": [{
+ "rule": [{
+ "metric_trigger": [{
+ "metric_name": "CpuPercentage",
+ "operator": "GreaterThan",
+ "statistic": "Average",
+ "threshold": 70,
+ "time_aggregation": "Average",
+ "time_grain": "PT1M",
+ "time_window": "PT5M"
+ }],
+ "scale_action": [{
+ "cooldown": "PT10M",
+ "direction": "Increase",
+ "type": "ChangeCount",
+ "value": 1
+ }]
+ }]
+ }]
+ }`)
+
+ k := "module.service_plan.azurerm_monitor_autoscale_setting.app_service_auto_scale"
+ description[k] = v
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/common.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/common.go
new file mode 100644
index 00000000..dac853cf
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/common.go
@@ -0,0 +1,23 @@
+package test
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/gruntwork-io/terratest/modules/random"
+)
+
+// these are useful values used in many test
+var region = "southcentralus"
+var prefix = "smpl" + strings.ToLower(random.UniqueId())
+var workspace = "smpl-" + strings.ToLower(random.UniqueId())
+
+// helper function to parse blocks of JSON into a generic Go map
+func asMap(t *testing.T, jsonString string) map[string]interface{} {
+ var theMap map[string]interface{}
+ if err := json.Unmarshal([]byte(jsonString), &theMap); err != nil {
+ t.Fatal(err)
+ }
+ return theMap
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/cosmosdb.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/cosmosdb.go
new file mode 100644
index 00000000..e6211ebb
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/cosmosdb.go
@@ -0,0 +1,39 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendCosmosDbTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedcosmosdbsqlcontainercollection := asMap(t, `{
+ "name" : "scoring-data",
+ "partition_key_path" : "/APPT_ID",
+ "throughput" : 40000
+ }`)
+ description["module.cosmosdb.azurerm_cosmosdb_sql_container.cosmos_collections[0]"] = expectedcosmosdbsqlcontainercollection
+
+ expectedschedulingcollection := asMap(t, `{
+ "name" : "scheduling-data",
+ "partition_key_path" : "/id",
+ "throughput" : 40000
+ }`)
+ description["module.cosmosdb.azurerm_cosmosdb_sql_container.cosmos_collections[1]"] = expectedschedulingcollection
+
+ expectedcosmosdbaccount := asMap(t, `{
+ "enable_automatic_failover" : false,
+ "enable_multiple_write_locations" : false,
+ "is_virtual_network_filter_enabled" : true,
+ "kind" : "GlobalDocumentDB",
+ "offer_type" : "Standard"
+ }`)
+ description["module.cosmosdb.azurerm_cosmosdb_account.main"] = expectedcosmosdbaccount
+
+ expectedcosmosdbsqldatabase := asMap(t, `{
+ "throughput" : 1000
+ }`)
+ description["module.cosmosdb.azurerm_cosmosdb_sql_database.cosmos_dbs[0]"] = expectedcosmosdbsqldatabase
+
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/datafactory.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/datafactory.go
new file mode 100644
index 00000000..e347d8bf
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/datafactory.go
@@ -0,0 +1,15 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendDataFactoryTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedDatafactoryApp := asMap(t, `{
+ "identity": [{ "type": "SystemAssigned" }]
+ }`)
+ description["module.data-factory.azurerm_data_factory.main"] = expectedDatafactoryApp
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/functionapp.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/functionapp.go
new file mode 100644
index 00000000..b220e428
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/functionapp.go
@@ -0,0 +1,19 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendFunctionAppTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedFunctionApp := asMap(t, `{
+ "enable_builtin_logging": true,
+ "enabled": true,
+ "https_only": false,
+ "version": "~3"
+ }`)
+
+ description["module.function_app.azurerm_function_app.main[0]"] = expectedFunctionApp
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/keyvault.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/keyvault.go
new file mode 100644
index 00000000..13f93bf0
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/keyvault.go
@@ -0,0 +1,58 @@
+package test
+
+import (
+ "fmt"
+ "sort"
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendKeyVaultTests(t *testing.T, description unit.ResourceDescription) {
+ kvBasicExpectations(t, description)
+ //kvAccessPolicyExpectations(t, description)
+ kvSecretExpectations(t, description)
+}
+
+func kvBasicExpectations(t *testing.T, description unit.ResourceDescription) {
+ k1 := "module.keyvault.azurerm_key_vault.keyvault"
+ e1 := asMap(t, `{
+ "sku_name": "standard"
+ }`)
+ description[k1] = e1
+}
+
+func kvSecretExpectations(t *testing.T, description unit.ResourceDescription) {
+ expectedKeys := []string{
+ "appinsights-key",
+ "mlw-appinsights-key",
+ "sys-storage-account-key",
+ "app-storage-account-key",
+ "app-sp-tenant-id",
+ "aad-client-id",
+ "app-sp-username",
+ "app-sp-password",
+ "cosmos-endpoint",
+ "cosmos-primary-key",
+ "cosmos-connection",
+ "function-app-key",
+ }
+
+ // The unit test fixture will expect these secrets to match ordinals returned by terraform, which sorts by name
+ sort.Strings(expectedKeys)
+ for index, value := range expectedKeys {
+ key := fmt.Sprintf("module.keyvault_secrets.azurerm_key_vault_secret.secret[%v]", index)
+ val := asMap(t, fmt.Sprintf(`{"name": "%s"}`, value))
+ description[key] = val
+ }
+}
+
+func kvAccessPolicyExpectations(t *testing.T, description unit.ResourceDescription) {
+ e1 := asMap(t, `{
+ "certificate_permissions": ["get", "list"],
+ "secret_permissions": ["get", "list"],
+ "key_permissions": ["get", "list"]
+ }`)
+ k1 := "module.app_service_keyvault_access_policy.azurerm_key_vault_access_policy.keyvault[0]"
+ description[k1] = e1
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/mlworkspace.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/mlworkspace.go
new file mode 100644
index 00000000..4ce5c6f4
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/mlworkspace.go
@@ -0,0 +1,17 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendMLWorkspaceTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedMLWorkspaceResult := map[string]interface{}{
+ "name": "demo0012",
+ "resource_group_name": "azmltest",
+ "sku_name": "Enterprise",
+ }
+ description["module.ml_workspace.azurerm_machine_learning_workspace.mlworkspace"] = expectedMLWorkspaceResult
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/storage.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/storage.go
new file mode 100644
index 00000000..2d342496
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/storage.go
@@ -0,0 +1,30 @@
+package test
+
+import (
+ "testing"
+
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+func appendStorageTests(t *testing.T, description unit.ResourceDescription) {
+
+ expectedStorageAccount := asMap(t, `{
+ "account_kind": "StorageV2",
+ "account_replication_type": "LRS",
+ "account_tier": "Standard"
+ }`)
+ description["module.sys_storage_account.azurerm_storage_account.main"] = expectedStorageAccount
+
+ expectedDataPrepStorageAccount := asMap(t, `{
+ "account_kind": "StorageV2",
+ "account_replication_type": "LRS",
+ "account_tier": "Standard"
+ }`)
+ description["module.app_storage_account.azurerm_storage_account.main"] = expectedDataPrepStorageAccount
+
+ expectedDataPrepStorageContainerList := asMap(t, `{
+ "container_access_type":"private",
+ "name": "sample"
+ }`)
+ description["module.app_storage_account.azurerm_storage_container.main[0]"] = expectedDataPrepStorageContainerList
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/tests/unit/unit_test.go b/infra/templates/az-svc-data-integration-mlw/tests/unit/unit_test.go
new file mode 100644
index 00000000..b96566c7
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/tests/unit/unit_test.go
@@ -0,0 +1,74 @@
+package test
+
+import (
+ "os"
+ "testing"
+
+ "github.com/gruntwork-io/terratest/modules/terraform"
+ "github.com/microsoft/terratest-abstraction/unit"
+)
+
+var tfOptions = &terraform.Options{
+ TerraformDir: "../../",
+ Upgrade: true,
+ Vars: map[string]interface{}{
+ "resource_group_location": region,
+ "prefix": prefix,
+ "app_services": []interface{}{
+ map[string]interface{}{
+ "app_name": "smpl",
+ "image": *new(string),
+ "app_command_line": *new(string),
+ "linux_fx_version": *new(string),
+ "app_settings": make(map[string]string, 0),
+ },
+ },
+ },
+ BackendConfig: map[string]interface{}{
+ "storage_account_name": os.Getenv("TF_VAR_remote_state_account"),
+ "container_name": os.Getenv("TF_VAR_remote_state_container"),
+ },
+}
+
+func TestTemplate(t *testing.T) {
+ expectedAppDevResourceGroup := asMap(t, `{
+ "location": "`+region+`"
+ }`)
+
+ expectedMlDevResourceGroup := asMap(t, `{
+ "location": "`+region+`"
+ }`)
+
+ expectedAppInsights := asMap(t, `{
+ "application_type": "web"
+ }`)
+
+ resourceDescription := unit.ResourceDescription{
+ "azurerm_resource_group.app_rg": expectedAppDevResourceGroup,
+ "module.app_insights.azurerm_application_insights.appinsights": expectedAppInsights,
+ }
+
+ resourceWorkspaceDescription := unit.ResourceDescription{
+ "azurerm_resource_group.mlw_rg": expectedMlDevResourceGroup,
+ "module.app_insights.azurerm_application_insights.appinsights": expectedAppInsights,
+ }
+
+ //appendAutoScaleTests(t, resourceDescription)
+ appendKeyVaultTests(t, resourceDescription)
+ appendStorageTests(t, resourceDescription)
+ appendMLWorkspaceTests(t, resourceWorkspaceDescription)
+ appendDataFactoryTests(t, resourceDescription)
+ appendFunctionAppTests(t, resourceDescription)
+ appendCosmosDbTests(t, resourceDescription)
+
+ testFixture := unit.UnitTestFixture{
+ GoTest: t,
+ TfOptions: tfOptions,
+ Workspace: workspace,
+ PlanAssertions: nil,
+ ExpectedResourceCount: 81,
+ ExpectedResourceAttributeValues: resourceDescription,
+ }
+
+ unit.RunUnitTests(&testFixture)
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/variables.tf b/infra/templates/az-svc-data-integration-mlw/variables.tf
new file mode 100644
index 00000000..7ee8d973
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/variables.tf
@@ -0,0 +1,286 @@
+// ---- General Configuration ----
+variable "prefix" {
+ description = "An identifier used to construct the names of all resources in this template."
+ type = string
+}
+
+variable "randomization_level" {
+ description = "Number of additional random characters to include in resource names to insulate against unexpected resource name collisions."
+ type = number
+ default = 8
+}
+
+variable "resource_group_location" {
+ description = "The Azure region where all resources in this template should be created."
+ type = string
+}
+
+variable "app_service_settings" {
+ description = "Map of app settings that will be applied across all provisioned app services."
+ type = map(string)
+ default = {}
+}
+
+variable "app_services" {
+ description = "Metadata about the app services to be created."
+ type = list(object({
+ app_name = string
+ image = string
+ linux_fx_version = string
+ app_command_line = string
+ app_settings = map(string)
+ }))
+}
+
+// ---- App Service Environment Configuration ----
+variable "ase_name" {
+ description = "The name of the App Service Environment in which to deploy the app"
+ type = string
+ default = ""
+}
+
+variable "address_space" {
+ description = "The address space that is used the virtual network. You can supply more than one address space. Changing this forces a new resource to be created"
+ type = list(string)
+ default = ["10.2.0.0/16"]
+}
+
+
+variable "subnet_address_prefix" {
+ description = "The subnet address prefix to use for the subnet"
+ type = string
+ default = "10.2.1.0/24"
+}
+
+variable "function_subnet_address_prefix" {
+ description = "The subnet address prefix to use for the subnet"
+ type = string
+ default = "10.2.2.0/24"
+}
+
+variable "subnet_service_endpoints" {
+ description = "A list of the service endpoints for the subnet"
+ type = list(string)
+ default = ["Microsoft.Web", "Microsoft.ContainerRegistry", "Microsoft.AzureCosmosDB"]
+}
+
+
+variable "func_subnet_service_endpoints" {
+ description = "A list of the service endpoints for the subnet"
+ type = list(string)
+ default = ["Microsoft.Web", "Microsoft.ContainerRegistry"]
+}
+
+variable "container_registry_sku" {
+ description = "(Optional) The SKU name of the the container registry. Possible values are Basic, Standard and Premium."
+ type = string
+ default = "Premium"
+}
+
+
+// ----Function App Service Plan Configuration ----
+
+variable "monitoring_dimension_values" {
+ description = "Dimensions used to determine service plan scaling"
+ type = list(string)
+ default = ["*"]
+}
+
+variable "func_app_service_plan_kind" {
+ description = "The kind of Service Plan to be created. Possible values are Windows/Linux/FunctionApp/App"
+ type = string
+ default = "FunctionApp"
+}
+
+
+variable "scaling_rules" {
+ description = "The scaling rules for the app service plan. Schema defined here: https://www.terraform.io/docs/providers/azurerm/r/monitor_autoscale_setting.html#rule. Note, the appropriate resource ID will be auto-inflated by the template"
+ type = list(object({
+ metric_trigger = object({
+ metric_name = string
+ time_grain = string
+ statistic = string
+ time_window = string
+ time_aggregation = string
+ operator = string
+ threshold = number
+ })
+ scale_action = object({
+ direction = string
+ type = string
+ cooldown = string
+ value = number
+ })
+ }))
+ default = [
+ {
+ metric_trigger = {
+ metric_name = "CpuPercentage"
+ time_grain = "PT1M"
+ statistic = "Average"
+ time_window = "PT5M"
+ time_aggregation = "Average"
+ operator = "GreaterThan"
+ threshold = 70
+ }
+ scale_action = {
+ direction = "Increase"
+ type = "ChangeCount"
+ value = 1
+ cooldown = "PT10M"
+ }
+ }
+ ]
+}
+
+variable "func_app_service_plan_size" {
+ description = "The size of the service plan instance."
+ type = string
+ default = "Y1"
+}
+
+variable "func_app_service_plan_tier" {
+ description = "The tier under which the service plan is created."
+ type = string
+ default = "Dynamic"
+}
+
+variable "func_app_service_plan_reserved" {
+ description = "Is the Service Plan to be created reserved. Possible values are true/false"
+ type = bool
+ default = true
+}
+
+// ----Function App Service Plan Configuration Ends----
+
+variable "sys_storage_containers" {
+ description = "The list of storage container names to create. Names must be unique per storage account."
+ type = list(string)
+}
+
+variable "app_storage_containers" {
+ description = "The list of storage container names to create. Names must be unique per storage account."
+ type = list(string)
+}
+
+
+// ---- ML Workspace Configuration ----
+
+variable "sku_name" {
+ description = "SKU/edition of the Machine Learning Workspace"
+ type = string
+ default = "Enterprise"
+}
+
+// ---- Monitoring Service Configuration ----
+
+variable "action_group_name" {
+ description = "The name of the action group."
+ type = string
+ default = "Simple Default Action Group"
+}
+
+variable "action_group_email_receiver" {
+ description = "The e-mail receiver for an alert rule resource."
+ type = string
+ default = ""
+}
+
+variable "metric_alert_name" {
+ description = "The display name of a group of metric alert criteria."
+ type = string
+ default = "smpl Metric Alerts"
+}
+
+variable "metric_alert_frequency" {
+ description = "The frequency with which the metric alert checks if the conditions are met."
+ type = string
+ default = "PT1M"
+}
+
+variable "metric_alert_period" {
+ description = "The look back window over which metric values are checked. Value must be greater than 'frequency'."
+ type = string
+ default = "PT5M"
+}
+
+variable "metric_alert_criteria_namespace" {
+ description = "A monitored resource namespace with configurable metric alert criteria."
+ type = string
+ default = "Microsoft.Web/serverfarms"
+}
+
+variable "metric_alert_criteria_name" {
+ description = "A predefined Azure resource alert monitoring rule name."
+ type = string
+ default = "CpuPercentage"
+}
+
+variable "metric_alert_criteria_aggregation" {
+ description = "The calculation used for building metric alert criteria."
+ type = string
+ default = "Average"
+}
+
+variable "metric_alert_criteria_operator" {
+ description = "A logical operator used for building metric alert criteria."
+ type = string
+ default = "GreaterThan"
+}
+
+variable "metric_alert_criteria_threshold" {
+ description = "The criteria threshold value that activates the metric alert."
+ type = number
+ default = 50
+}
+
+// ---- Function App Service Configuration ----
+
+variable "fn_name_prefix" {
+ description = "The function app prefix name"
+ type = string
+ default = "smpl-test"
+}
+
+variable "fn_app_config" {
+ description = "Metadata about the app services to be created"
+ type = map(object({
+ image = string,
+ zip = string,
+ hash = string
+ }))
+ default = {}
+}
+
+// ---- Azure Data Factory Configuration ----
+
+variable "adf_number_of_nodes" {
+ description = "Number of nodes for the Managed Integration Runtime. Max is 10. Defaults to 1."
+ type = number
+ default = 1
+}
+
+variable "adf_max_parallel_executions_per_node" {
+ description = " Defines the maximum parallel executions per node. Defaults to 1. Max is 16."
+ type = number
+ default = 1
+}
+
+variable "adf_trigger_interval" {
+ description = "The interval for how often the trigger occurs. This defaults to 1."
+ type = number
+ default = 1
+}
+
+variable "adf_trigger_frequency" {
+ description = "The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute."
+ type = string
+ default = "Minute"
+}
+
+
+// ---- Cosmos Database Configuration ----
+variable "primary_replica_location" {
+ description = "The primary replica location for the cosmos database"
+ type = string
+}
diff --git a/infra/templates/az-svc-data-integration-mlw/version.tf b/infra/templates/az-svc-data-integration-mlw/version.tf
new file mode 100644
index 00000000..d9b6f790
--- /dev/null
+++ b/infra/templates/az-svc-data-integration-mlw/version.tf
@@ -0,0 +1,3 @@
+terraform {
+ required_version = ">= 0.12"
+}