diff --git a/docs/wiki/Frequently-Asked-Questions.md b/docs/wiki/Frequently-Asked-Questions.md index 5a13d2dc..2a9e38cd 100644 --- a/docs/wiki/Frequently-Asked-Questions.md +++ b/docs/wiki/Frequently-Asked-Questions.md @@ -158,6 +158,8 @@ Yes, ensure the following setting combinations are applied (replace `rgname1`, ` "Core.SkipResourceGroup": false + "Core.SubscriptionsToIncludeChildResource": ["SubscriptionId1","SubscriptionId2"] + "Core.SubscriptionsToIncludeResourceGroups": ["SubscriptionId1","SubscriptionId2"] ``` diff --git a/docs/wiki/Settings.md b/docs/wiki/Settings.md index 17ab7e8b..c6dd2a43 100644 --- a/docs/wiki/Settings.md +++ b/docs/wiki/Settings.md @@ -35,14 +35,15 @@ The following configuration values can be modified within the `settings.json` fi | 23 | SkipResourceType | Skip specific [Resource Types](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types) (only targets Resource Group scoped resources) | `"Core.SkipResourceType": ["Microsoft.VSOnline/plans"]` | | 24 | SkipRole | Do not include Role types in pull | `"Core.SkipRole": false` | | 25 | State | Folder to store AzOpsState artefact, defaults to `root` | `"Core.State: "/root"` | -| 26 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | -| 27 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | -| 28 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | -| 29 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | -| 30 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | -| 31 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | -| 32 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | -| 33 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | +| 26 | SubscriptionsToIncludeChildResource | Filter which Subscription IDs should include child resources in pull | `"Core.SubscriptionsToIncludeChildResource": ["*"]` | +| 27 | SubscriptionsToIncludeResourceGroups | Filter which Subscription IDs should include Resource Groups in pull [Logic Updated in v2.0.0](https://github.com/Azure/AzOps/releases/tag/2.0.0) | `"Core.SubscriptionsToIncludeResourceGroups": ["*"]` | +| 28 | TemplateParameterFileSuffix | Default template file suffix. *Not recommended to change* | `"Core.TemplateParameterFileSuffix": ".json"` | +| 29 | AllowMultipleTemplateParameterFiles | Control multiple parameter file behaviour. *Not recommended to change* | `"Core.AllowMultipleTemplateParameterFiles": false` | +| 30 | DeployAllMultipleTemplateParameterFiles | Control base template deployment behaviour with changes and un-changed multiple corresponding parameter files. | `"Core.DeployAllMultipleTemplateParameterFiles": false` | +| 31 | MultipleTemplateParameterFileSuffix | Multiple parameter file suffix identifier. *Example mytemplate.x1.bicepparam* | `"Core.MultipleTemplateParameterFileSuffix": ".x"` | +| 32 | ParallelDeployMultipleTemplateParameterFiles | Control parallel deployment of MultipleTemplateParameterFiles behaviour | `"Core.ParallelDeployMultipleTemplateParameterFiles": false` | +| 33 | ThrottleLimit | Value declaring number of parallel threads. [Read more](https://github.com/azure/azops/wiki/performance-considerations) | `"Core.ThrottleLimit": 5` | +| 34 | WhatifExcludedChangeTypes | Exclude specific change types from WhatIf operations | `"Core.WhatifExcludedChangeTypes": ["NoChange","Ignore"]` | ## Workflow / Pipeline Settings diff --git a/src/internal/configurations/Core.ps1 b/src/internal/configurations/Core.ps1 index ad5a98ce..3bb636af 100644 --- a/src/internal/configurations/Core.ps1 +++ b/src/internal/configurations/Core.ps1 @@ -14,7 +14,7 @@ Set-PSFConfig -Module AzOps -Name Core.CustomJqTemplatePath -Value (Join-Path $p Set-PSFConfig -Module AzOps -Name Core.SkipCustomJqTemplate -Value $true -Initialize -Validation bool -Description 'Controls usage of CustomJqTemplatePath to search for custom jq template' Set-PSFConfig -Module AzOps -Name Core.MainTemplate -Value "$script:ModuleRoot\data\template\template.json" -Initialize -Validation string -Description 'Main template json' Set-PSFConfig -Module AzOps -Name Core.OfferType -Value 'MS-AZR-0017P' -Initialize -Validation string -Description '-' -Set-PSFConfig -Module AzOps -Name Core.PartialMgDiscoveryRoot -Value @() -Initialize -Validation stringarray -Description 'Used in combination with AZOPS_SUPPORT_PARTIAL_MG_DISCOVERY, example value: "Contoso","Tailspin","Management"' +Set-PSFConfig -Module AzOps -Name Core.PartialMgDiscoveryRoot -Value @() -Initialize -Validation stringarray -Description 'Generate folder hierachy for specific Management Groups IDs' Set-PSFConfig -Module AzOps -Name Core.IncludeResourcesInResourceGroup -Value @('*') -Initialize -Validation stringarray -Description 'Global flag to discover only resources in these resource groups.' Set-PSFConfig -Module AzOps -Name Core.IncludeResourceType -Value @('*') -Initialize -Validation stringarray -Description 'Global flag to discover only specific resource types.' Set-PSFConfig -Module AzOps -Name Core.SkipChildResource -Value $true -Initialize -Validation bool -Description 'Global flag to indicate whether child resources should be discovered or not. Requires SkipResourceGroup and SkipResource to be false.' @@ -26,7 +26,8 @@ Set-PSFConfig -Module AzOps -Name Core.SkipResourceGroup -Value $false -Initiali Set-PSFConfig -Module AzOps -Name Core.SkipResourceType -Value @('Microsoft.VSOnline/plans', 'Microsoft.PowerPlatform/accounts', 'Microsoft.PowerPlatform/enterprisePolicies') -Initialize -Validation stringarray -Description 'Global flag to skip discovery of specific Resource types.' Set-PSFConfig -Module AzOps -Name Core.SkipRole -Value $false -Initialize -Validation bool -Description '-' Set-PSFConfig -Module AzOps -Name Core.State -Value (Join-Path $pwd -ChildPath "root") -Initialize -Validation string -Description 'Folder to store AzOpsState artefact' -Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID or Display Name that matches the filter. Powershell filter that matches with like operator is supported.' +Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeChildResource -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup, SkipResource and SkipChildResource to be false. Subscription ID that matches the filter.' +Set-PSFConfig -Module AzOps -Name Core.SubscriptionsToIncludeResourceGroups -Value @('*') -Initialize -Validation stringarray -Description 'Requires SkipResourceGroup to be false. Subscription ID that matches the filter.' Set-PSFConfig -Module AzOps -Name Core.TemplateParameterFileSuffix -Value '.json' -Initialize -Validation string -Description 'Parameter file suffix identifier' Set-PSFConfig -Module AzOps -Name Core.AllowMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control multiple parameter file behaviour' Set-PSFConfig -Module AzOps -Name Core.DeployAllMultipleTemplateParameterFiles -Value $false -Initialize -Validation string -Description 'Global flag to control base template deployment behaviour with changes and un-changed multiple corresponding parameter files' diff --git a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 index 9e7bbc92..59b67af4 100644 --- a/src/internal/functions/Get-AzOpsResourceDefinition.ps1 +++ b/src/internal/functions/Get-AzOpsResourceDefinition.ps1 @@ -29,6 +29,10 @@ Skip discovery of roles for better performance. .PARAMETER StatePath The root folder under which to write the resource json. + .PARAMETER SubscriptionsToIncludeChildResource + Filter which Subscription IDs should include child resources in pull. + .PARAMETER SubscriptionsToIncludeResourceGroups + Filter which Subscription IDs should include Resource Groups in pull. .EXAMPLE $TenantRootId = '/providers/Microsoft.Management/managementGroups/{0}' -f (Get-AzTenant).Id Get-AzOpsResourceDefinition -scope $TenantRootId -Verbose @@ -84,7 +88,13 @@ [Parameter(Mandatory = $false)] [string] - $StatePath = (Get-PSFConfigValue -FullName 'AzOps.Core.State') + $StatePath = (Get-PSFConfigValue -FullName 'AzOps.Core.State'), + + [string[]] + $SubscriptionsToIncludeChildResource = (Get-PSFConfigValue -FullName 'AzOps.Core.SubscriptionsToIncludeChildResource'), + + [string[]] + $SubscriptionsToIncludeResourceGroups = (Get-PSFConfigValue -FullName 'AzOps.Core.SubscriptionsToIncludeResourceGroups') ) begin { @@ -222,12 +232,15 @@ } } else { - if ((Get-PSFConfigValue -FullName 'AzOps.Core.SubscriptionsToIncludeResourceGroups') -ne '*') { - $subscriptionsToIncludeResourceGroups = $subscriptions | Where-Object { $_.Id -in (Get-PSFConfigValue -FullName 'AzOps.Core.SubscriptionsToIncludeResourceGroups') } - } $query = "resourcecontainers | where type == 'microsoft.resources/subscriptions/resourcegroups' | where managedBy == '' | order by ['id'] asc" - if ($subscriptionsToIncludeResourceGroups) { - $resourceGroups = Search-AzOpsAzGraph -Subscription $subscriptionsToIncludeResourceGroups -Query $query -ErrorAction Stop + if ($SubscriptionsToIncludeResourceGroups -ne '*') { + $newSubscriptionsToIncludeResourceGroups = $subscriptions | Where-Object { $_.Id -in $SubscriptionsToIncludeResourceGroups } + if ($newSubscriptionsToIncludeResourceGroups) { + $resourceGroups = Search-AzOpsAzGraph -Subscription $newSubscriptionsToIncludeResourceGroups -Query $query -ErrorAction Stop + } + else { + Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.Subscription.NotFound' -Target $ScopeObject + } } else { $resourceGroups = Search-AzOpsAzGraph -Subscription $subscriptions -Query $query -ErrorAction Stop @@ -280,8 +293,8 @@ } # Process Policies at Resource Group scope if (-not $SkipPolicy) { - if ($subscriptionsToIncludeResourceGroups) { - Get-AzOpsPolicy -ScopeObject $scopeObject -Subscription $subscriptions -SubscriptionsToIncludeResourceGroups $subscriptionsToIncludeResourceGroups -ResourceGroup -StatePath $StatePath + if ($newSubscriptionsToIncludeResourceGroups) { + Get-AzOpsPolicy -ScopeObject $scopeObject -Subscription $subscriptions -SubscriptionsToIncludeResourceGroups $newSubscriptionsToIncludeResourceGroups -ResourceGroup -StatePath $StatePath } else { Get-AzOpsPolicy -ScopeObject $scopeObject -Subscription $subscriptions -ResourceGroup -StatePath $StatePath @@ -324,8 +337,11 @@ else { Write-AzOpsMessage -LogLevel Debug -LogString 'Get-AzOpsResourceDefinition.SkippingResources' -Target $ScopeObject } - # Process resources as scope in parallel, look for childResource + # Process Child resources at resource scope in parallel if (-not $SkipResource -and -not $SkipChildResource) { + if ($SubscriptionsToIncludeChildResource -ne '*') { + $resources = $resources | Where-Object { $_.subscriptionId -in $SubscriptionsToIncludeChildResource } + } $resources | Foreach-Object -ThrottleLimit (Get-PSFConfigValue -FullName 'AzOps.Core.ThrottleLimit') -Parallel { $resource = $_ $runspaceData = $using:runspaceData diff --git a/src/tests/functional/Functional.Tests.ps1 b/src/tests/functional/Functional.Tests.ps1 index a6e72e53..21829709 100644 --- a/src/tests/functional/Functional.Tests.ps1 +++ b/src/tests/functional/Functional.Tests.ps1 @@ -68,7 +68,8 @@ if (Test-Path -Path $generatedRoot) { # Invoke the Invoke-AzOpsPull function to generate the scope data which can be tested against to ensure structure is correct and data model hasn't changed. -Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeResourceGroups -Value $script:subscriptionId +Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeChildResource -Value @($script:subscriptionId) +Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeResourceGroups -Value @($script:subscriptionId) Set-PSFConfig -FullName AzOps.Core.SkipChildResource -Value $false Set-PSFConfig -FullName AzOps.Core.SkipPim -Value $false $deploymentLocationId = (Get-FileHash -Algorithm SHA256 -InputStream ([IO.MemoryStream]::new([byte[]][char[]](Get-PSFConfigValue -FullName 'AzOps.Core.DefaultDeploymentRegion')))).Hash.Substring(0, 4) diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index 108acce9..e8d3858f 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -55,8 +55,9 @@ Describe "Repository" { Write-PSFMessage -Level Verbose -Message "Creating repository test environment" -FunctionName "BeforeAll" $templateFile = Join-Path -Path $global:testroot -ChildPath "templates/azuredeploy.jsonc" $templateParameters = @{ - "tenantId" = "$script:tenantId" - "subscriptionId" = "$script:subscriptionId" + "tenantId" = $script:tenantId + "subscriptionId" = $script:subscriptionId + "otherSubscriptionId" = $otherSubscription[0].Id } $params = @{ ManagementGroupId = "$script:tenantId" @@ -164,7 +165,8 @@ Describe "Repository" { #endregion PartialMgDiscoveryRoot Pull #region GeneratedRoot Pull - Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeResourceGroups -Value $script:subscriptionId + Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeChildResource -Value @($script:subscriptionId) + Set-PSFConfig -FullName AzOps.Core.SubscriptionsToIncludeResourceGroups -Value @($script:subscriptionId,$otherSubscription[0].Id) Set-PSFConfig -FullName AzOps.Core.PartialMgDiscoveryRoot -Value @() Set-PSFConfig -FullName AzOps.Core.State -Value $generatedRoot Set-PSFConfig -FullName AzOps.Core.SkipLock -Value $false @@ -287,7 +289,7 @@ Describe "Repository" { $script:subscriptionFile = ($script:subscriptionPath).FullName Write-PSFMessage -Level Debug -Message "SubscriptionFile: $($script:subscriptionFile)" -FunctionName "BeforeAll" - $script:resourceGroupPath = ($filePaths | Where-Object Name -eq "microsoft.resources_resourcegroups-$(($script:resourceGroup.ResourceGroupName).toLower()).json") + $script:resourceGroupPath = ($filePaths | Where-Object { $_.Name -eq "microsoft.resources_resourcegroups-$(($script:resourceGroup.ResourceGroupName).toLower()).json" -and $_.FullName -match $script:subscriptionId }) $script:resourceGroupDirectory = ($script:resourceGroupPath).Directory $script:resourceGroupFile = ($script:resourceGroupPath).FullName $script:resourceGroupDeploymentName = "AzOps-{0}-{1}" -f $($script:resourceGroupPath.Name.Replace(".json", '')), $deploymentLocationId @@ -1025,6 +1027,9 @@ Describe "Repository" { $script:ruleCollectionDeployment = Get-AzResourceGroupDeployment -ResourceGroupName 'App1-azopsrg' -Name $script:ruleCollectionDeploymentName $ruleCollectionDeployment.ProvisioningState | Should -Be "Succeeded" } + It "Validate SubscriptionsToIncludeChildResource filter by ensuring only one rulegroup child item is pulled back" { + $script:ruleCollectionGroupsPath.Count | Should -Be 1 + } #endregion #region Scope - logAnalyticsWorkspaceSavedSearchesPath (./root/tenant root group/test/platform/management/subscription-0/App1-azopsrg/thisisalongloganalyticsworkspacename123456789011121314151617181) diff --git a/src/tests/templates/azuredeploy.jsonc b/src/tests/templates/azuredeploy.jsonc index 5d64d707..25fa645a 100644 --- a/src/tests/templates/azuredeploy.jsonc +++ b/src/tests/templates/azuredeploy.jsonc @@ -7,6 +7,9 @@ }, "subscriptionId": { "type": "string" + }, + "otherSubscriptionId": { + "type": "string" } }, "functions": [], @@ -827,6 +830,138 @@ } } }, + // Resource Group, Role Assignment, Policy Exemption and Route Table - Test - Platform - Management - otherSubscription-x - App1-azopsrg + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "AzOps-Tests-otherNested", + "subscriptionId": "[parameters('otherSubscriptionId')]", + "location": "northeurope", + "dependsOn": [], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2019-10-01", + "name": "App1-azopsrg", + "location": "northeurope" + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2019-10-01", + "name": "RouteTableAndPolicy", + "resourceGroup": "App1-azopsrg", + "dependsOn": [ + "App1-azopsrg" + ], + "properties": { + "mode": "Incremental", + "expressionEvaluationOptions": { + "scope": "inner" + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/firewallPolicies", + "name": "TestPolicy", + "apiVersion": "2021-06-01", + "location": "eastus", + "tags": {}, + "properties": { + "sku": { + "tier": "Standard" + }, + "threatIntelMode": "Alert", + "threatIntelWhitelist": { + "fqdns": [], + "ipAddresses": [] + }, + "childPolicies": [], + "ruleCollectionGroups": [ + ], + "firewalls": [] + } + } + ], + "outputs": { + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2019-10-01", + "name": "RuleGroupCollection", + "resourceGroup": "App1-azopsrg", + "dependsOn": [ + "RouteTableAndPolicy" + ], + "properties": { + "mode": "Incremental", + "expressionEvaluationOptions": { + "scope": "inner" + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/firewallPolicies/ruleCollectionGroups", + "apiVersion": "2020-11-01", + "name": "TestPolicy/TestGroup", + "location": "eastus", + "properties": { + "priority": 100, + "ruleCollections": [ + { + "ruleCollectionType": "FirewallPolicyFilterRuleCollection", + "action": { + "type": "Allow" + }, + "rules": [ + { + "ruleType": "NetworkRule", + "name": "Test", + "ipProtocols": [ + "TCP" + ], + "sourceAddresses": [ + "10.100.100.0/24" + ], + "sourceIpGroups": [], + "destinationAddresses": [ + "10.200.201.0/24" + ], + "destinationIpGroups": [], + "destinationFqdns": [], + "destinationPorts": [ + "3389" + ] + } + ], + "name": "TestRule", + "priority": 101 + } + ] + } + } + + ], + "outputs": { + } + } + } + } + ] + } + } + }, // Management Group - Test - Platform - Identity { "type": "Microsoft.Management/managementGroups",