Create private endpoint for Data Factory when required by compliance …

…framework (#124) * Add support for private endpoint on Data Factory * Code cleanup
Azure · Sep 6, 2024 · c0041b3 · c0041b3
1 parent da01fea
commit c0041b3
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 38 deletions.
diff --git a/research-spoke/main.bicep b/research-spoke/main.bicep
@@ -583,7 +583,6 @@ module airlockModule './spoke-modules/airlock/main.bicep' = {
     keyVaultResourceGroupName: securityRg.name
 
     namingStructure: namingStructure
-    privateStorageAccountConnStringSecretName: privateStorageConnStringSecretModule.outputs.secretName
     spokePrivateStorageAccountName: storageModule.outputs.storageAccountName
     publicStorageAccountAllowedIPs: publicStorageAccountAllowedIPs
 
@@ -596,10 +595,9 @@ module airlockModule './spoke-modules/airlock/main.bicep' = {
 
     researcherAadObjectId: researcherEntraIdObjectId
 
-    // TODO: Only if usePrivateEndpoint is true
-    privateDnsZonesResourceGroupId: hubPrivateDnsZonesResourceGroupId
+    privateDnsZonesResourceGroupId: usePrivateEndpoints ? hubPrivateDnsZonesResourceGroupId : ''
     // If airlock review is centralized, then we don't need to create a private endpoint because we don't create a storage account
-    privateEndpointSubnetId: !isAirlockReviewCentralized
+    privateEndpointSubnetId: !isAirlockReviewCentralized && usePrivateEndpoints
       ? networkModule.outputs.createdSubnets.privateEndpointSubnet.id
       : ''
 
@@ -619,6 +617,8 @@ module airlockModule './spoke-modules/airlock/main.bicep' = {
     storageAccountRoleAssignments: [
       storageAccountReaderRoleAssignmentForResearcherGroup
     ]
+
+    usePrivateEndpoints: usePrivateEndpoints
   }
 }
 

diff --git a/research-spoke/spoke-modules/airlock/adf.bicep b/research-spoke/spoke-modules/airlock/adf.bicep
@@ -7,7 +7,10 @@ param subWorkloadName string
 param privateStorageAcctName string
 param userAssignedIdentityId string
 param userAssignedIdentityPrincipalId string
-param privateStorageAccountConnStringSecretName string
+
+param usePrivateEndpoint bool
+param privateEndpointSubnetId string
+param privateDnsZonesResourceGroupId string
 
 param roles object
 
@@ -60,17 +63,77 @@ resource adf 'Microsoft.DataFactory/factories@2018-06-01' = {
         userAssignedIdentity: encryptionUserAssignedIdentityId
       }
     }
+
+    publicNetworkAccess: usePrivateEndpoint ? 'Disabled' : 'Enabled'
+  }
+  tags: tags
+}
+
+var dataFactoryPrivateEndpointName = replace(baseName, '{rtype}', 'pe-adf')
+
+resource dataFactoryPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = if (usePrivateEndpoint) {
+  name: dataFactoryPrivateEndpointName
+  location: location
+  properties: {
+    subnet: {
+      id: privateEndpointSubnetId
+    }
+    privateLinkServiceConnections: [
+      {
+        name: dataFactoryPrivateEndpointName
+        properties: {
+          privateLinkServiceId: adf.id
+          groupIds: [
+            'dataFactory'
+          ]
+        }
+      }
+    ]
   }
   tags: tags
 }
 
+// az.environment() doesn't have this as of 2024-01-05
+var dataFactoryFqdns = {
+  AzureCloud: 'datafactory.azure.net'
+  AzureUSGovernment: 'datafactory.azure.us'
+}
+
+var dataFactoryFqdn = dataFactoryFqdns[az.environment().name]
+
+var privateDnsZonesSubscriptionId = usePrivateEndpoint ? split(privateDnsZonesResourceGroupId, '/')[2] : ''
+var privateDnsZonesResourceGroupName = usePrivateEndpoint ? split(privateDnsZonesResourceGroupId, '/')[4] : ''
+
+resource dataFactoryPrivateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-01-01' = if (usePrivateEndpoint) {
+  name: 'default'
+  parent: dataFactoryPrivateEndpoint
+  properties: {
+    privateDnsZoneConfigs: [
+      {
+        // Replace . with - because the config name doesn't suppport .
+        name: 'privatelink-${replace(dataFactoryFqdn, '.', '-')}'
+        properties: {
+          privateDnsZoneId: resourceId(
+            privateDnsZonesSubscriptionId,
+            privateDnsZonesResourceGroupName,
+            'Microsoft.Network/privateDnsZones',
+            'privatelink.${dataFactoryFqdn}'
+          )
+        }
+      }
+    ]
+  }
+}
+
 // Assign roles to UAMI to enable making modifications to ADF (start, stop) and approve private endpoints
+// TODO: Use AVM-style role assignment
 resource adfRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
-  name: guid(adf.id, userAssignedIdentityPrincipalId, roles['Data Factory Contributor'])
+  name: guid(adf.id, userAssignedIdentityPrincipalId, roles.DataFactoryContributor)
   scope: adf
   properties: {
-    roleDefinitionId: roles['Data Factory Contributor']
+    roleDefinitionId: roles.DataFactoryContributor
     principalId: userAssignedIdentityPrincipalId
+    principalType: 'ServicePrincipal'
   }
 }
 
@@ -115,21 +178,6 @@ resource integrationRuntime 'Microsoft.DataFactory/factories/integrationRuntimes
   }
 }
 
-// RBAC role assignment for ADF to Key Vault
-// HACK: 2024-08-15: ADF is granted permissions to the entire Key Vault using Key Vault Secrets User role.
-//         This is no longer needed.
-// module adfKeyVaultRoleAssignmentModule '../../../module-library/roleAssignments/roleAssignment-kv-secret.bicep' = {
-//   name: take(replace(deploymentNameStructure, '{rtype}', 'rbac-adf-kv-st'), 64)
-//   scope: resourceGroup(keyVaultResourceGroupName)
-//   params: {
-//     principalId: adf.identity.principalId
-//     roleDefinitionId: roles.KeyVaultSecretsUser
-//     kvName: keyVault.name
-//     secretName: privateStorageAccountConnStringSecretName
-//     principalType: 'ServicePrincipal'
-//   }
-// }
-
 // Grant ADF managed identity access to project/spoke Key Vault to retrieve secrets (#12)
 module adfPrjKvRoleAssignmentModule '../../../module-library/roleAssignments/roleAssignment-kv.bicep' = {
   name: replace(deploymentNameStructure, '{rtype}', 'adf-role-prjkv')

diff --git a/research-spoke/spoke-modules/airlock/main.bicep b/research-spoke/spoke-modules/airlock/main.bicep
@@ -32,8 +32,6 @@ param roles object
 param keyVaultName string
 param keyVaultResourceGroupName string
 
-param privateStorageAccountConnStringSecretName string
-
 @description('The name of the file share used for Airlock export reviews. The same parameter is used regardless of whether the airlock review is centralized or not.')
 param airlockFileShareName string
 
@@ -52,6 +50,7 @@ param adfEncryptionKeyName string
 
 param privateDnsZonesResourceGroupId string
 param privateEndpointSubnetId string
+param usePrivateEndpoints bool
 
 param domainJoinSpokeAirlockStorageAccount bool
 param domainJoinInfo activeDirectoryDomainInfo = {
@@ -222,26 +221,16 @@ module adfModule 'adf.bicep' = {
     keyVaultName: keyVaultName
     keyVaultResourceGroupName: keyVaultResourceGroupName
 
-    privateStorageAccountConnStringSecretName: privateStorageAccountConnStringSecretName
     adfEncryptionKeyName: adfEncryptionKeyName
     encryptionUserAssignedIdentityId: encryptionUamiId
     encryptionKeyVaultUri: encryptionKeyVaultUri
+
+    usePrivateEndpoint: usePrivateEndpoints
+    privateEndpointSubnetId: usePrivateEndpoints ? privateEndpointSubnetId : ''
+    privateDnsZonesResourceGroupId: usePrivateEndpoints ? privateDnsZonesResourceGroupId : ''
   }
 }
 
-// HACK: 2024-08-15: Moved to ADF module where there was already some duplication
-// // Grant ADF managed identity access to project/spoke Key Vault to retrieve secrets (#12)
-// module adfPrjKvRoleAssignmentModule '../../../module-library/roleAssignments/roleAssignment-kv.bicep' = {
-//   name: replace(deploymentNameStructure, '{rtype}', 'adf-role-prjkv')
-//   scope: spokeKeyVaultRg
-//   params: {
-//     kvName: keyVault.name
-//     principalId: adfModule.outputs.principalId
-//     roleDefinitionId: roles.KeyVaultSecretsUser
-//     principalType: 'ServicePrincipal'
-//   }
-// }
-
 var airlockStorageAccountName = useCentralizedReview
   ? centralizedModule.outputs.centralAirlockStorageAccountName
   : spokeAirlockStorageAccountModule.outputs.storageAccountName