diff --git a/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml new file mode 100644 index 00000000000..cbe0c6bb6af --- /dev/null +++ b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml @@ -0,0 +1,73 @@ +parameters: +- name: AdditionalParameters + type: object +- name: CloudConfig + type: object + default: {} +- name: MatrixConfigs + type: object + default: [] +- name: MatrixFilters + type: object + default: [] +- name: JobTemplatePath + type: string + +jobs: +- job: generate_matrix + variables: + displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ] + pool: + name: Azure Pipelines + vmImage: ubuntu-18.04 + displayName: Generate Job Matrix + steps: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter "$(displayNameFilter)" + -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}" + -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}" + displayName: Generate VM Job Matrix ${{ config.Name }} + name: generate_vm_job_matrix_${{ config.Name }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter "$(displayNameFilter)" + -Filters "${{ join('","', parameters.MatrixFilters) }}", "container=.*", "SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}" + -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}" + displayName: Generate Container Job Matrix + name: generate_container_job_matrix_${{ config.Name }} + +- ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix'] + DependsOn: generate_matrix + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: true + Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix'] + DependsOn: generate_matrix + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 new file mode 100644 index 00000000000..3e075c3e04e --- /dev/null +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -0,0 +1,35 @@ +<# + .SYNOPSIS + Generates a JSON object representing an Azure Pipelines Job Matrix. + See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec + + .EXAMPLE + ./eng/common/scripts/Create-JobMatrix $context +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$True)][string] $ConfigPath, + [Parameter(Mandatory=$True)][string] $Selection, + [Parameter(Mandatory=$False)][string] $DisplayNameFilter, + [Parameter(Mandatory=$False)][array] $Filters, + [Parameter(Mandatory=$False)][array] $NonSparseParameters +) + +. $PSScriptRoot/job-matrix-functions.ps1 + +$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +# Strip empty string filters in order to be able to use azure pipelines yaml join() +$Filters = $Filters | Where-Object { $_ } + +[array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -nonSparseParameters $NonSparseParameters + +$serialized = SerializePipelineMatrix $matrix + +Write-Output $serialized.pretty +Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md new file mode 100644 index 00000000000..4f468e4ccb3 --- /dev/null +++ b/eng/common/scripts/job-matrix/README.md @@ -0,0 +1,461 @@ +# Azure Pipelines Matrix Generator + +* [Usage in a pipeline](#usage-in-a-pipeline) +* [Matrix config file syntax](#matrix-config-file-syntax) + * [Fields](#fields) + * [matrix](#matrix) + * [include](#include) + * [exclude](#exclude) + * [displayNames](#displaynames) + * [$IMPORT](#import) +* [Matrix Generation behavior](#matrix-generation-behavior) + * [all](#all) + * [sparse](#sparse) + * [include/exclude](#includeexclude) + * [displayNames](#displaynames-1) + * [Filters](#filters) + * [NonSparseParameters](#nonsparseparameters) + * [Under the hood](#under-the-hood) +* [Testing](#testing) + + +This directory contains scripts supporting dynamic, cross-product matrix generation for azure pipeline jobs. +It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs), +but also adds some additional features like sparse matrix generation, cross-product includes and excludes, and programmable matrix filters. + +This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input +for a job matrix definition](https://docs.microsoft.com/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section). + +## Usage in a pipeline + +In order to use these scripts in a pipeline, you must provide a config file and call the matrix creation script within a powershell job. + +For a single matrix, you can include the `/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml` template in a pipeline (see /eng/common/scripts/job-matrix/samples/matrix-test.yml for a full working example): + +``` +jobs: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + MatrixConfigs: + - Name: base_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: all + NonSparseParameters: + - framework + GenerateVMJobs: true + - Name: sparse_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: sparse + GenerateVMJobs: true + JobTemplatePath: /eng/common/scripts/job-matrix/samples/matrix-job-sample.yml + AdditionalParameters: [] + CloudConfig: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Location: eastus2 + Cloud: Public + MatrixFilters: [] +``` + +## Matrix config file syntax + +Matrix parameters can either be a list of strings, or a set of grouped strings (represented as a hash). The latter parameter +type is useful for when 2 or more parameters need to be grouped together, but without generating more than one matrix permutation. + +``` +"matrix": { + "": [ ], + "": [ ], + "": { + "": { + "", + }, + "": { + "", + } + } +} +"include": [ , , ... ], +"exclude": [ , , ... ], +"displayNames": { : } +``` + +See `samples/matrix.json` for a full sample. + +### Fields + +#### matrix + +The `matrix` field defines the base cross-product matrix. The generated matrix can be full or sparse. + +Example: +``` +"matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ], + "additionalTestArguments": [ + "", + "/p:UseProjectReferenceToAzureClients=true", + ] +} +``` + +#### include + +The `include` field defines any number of matrices to be appended to the base matrix after processing exclusions. + +#### exclude + +The `include` field defines any number of matrices to be removed from the base matrix. Exclude parameters can be a partial +set, meaning as long as all exclude parameters match against a matrix entry (even if the matrix entry has additional parameters), +then it will be excluded from the matrix. For example, the below entry will match the exclusion and be removed: + +``` +matrix entry: +{ + "a": 1, + "b": 2, + "c": 3, +} + +"exclude": [ + { + "a": 1, + "b": 2 + } +] +``` + +#### displayNames + +Specify any overrides for the azure pipelines definition and UI that determines the matrix job name. If some parameter +values are too long or unreadable for this purpose (e.g. a command line argument), then you can replace them with a more +readable value here. For example: + +``` +"displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" +}, +"matrix": { + "additionalTestArguments": [ + "/p:UseProjectReferenceToAzureClients=true" + ] +} +``` + +#### $IMPORT + +Matrix configs can also import another matrix config. The effect of this is the imported matrix will be generated, +and then the importing config will be combined with that matrix (as if each entry of the imported matrix was a parameter). +To import a matrix, add a parameter with the key `$IMPORT`: + +``` +"matrix": { + "$IMPORT": "path/to/matrix.json", + "JavaVersion": [ "1.8", "1.11" ] +} +``` + +Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it +once for each instance of a language version. + +The processing order is as follows: + +Given a matrix and import matrix like below: +``` +{ + "matrix": { + "$IMPORT": "example-matrix.json", + "endpointType": [ "storage", "cosmos" ], + "JavaVersion": [ "1.8", "1.11" ] + }, + "include": [ + { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ] +} + +### example-matrix.json to import +{ + "matrix": { + "operatingSystem": [ "windows", "linux" ], + "client": [ "netty", "okhttp" ] + }, + "include": [ + { + "operatingSystem": "mac", + "client": "netty" + } + ] +} +``` + +1. The base matrix is generated (sparse in this example): + ``` + { + "storage_18": { + "endpointType": "storage", + "JavaVersion": "1.8" + }, + "cosmos_111": { + "endpointType": "cosmos", + "JavaVersion": "1.11" + } + } + ``` +1. The imported base matrix is generated (sparse in this example): + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + } + } + ``` +1. Includes/excludes from the imported matrix get applied to the imported matrix + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + }, + "mac_netty": { + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. The base matrix is multipled by the imported matrix (in this case, the base matrix has 2 elements, and the imported + matrix has 3 elements, so the product is a matrix with 6 elements: + ``` + "storage_18_windows_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "windows", + "client": "netty" + }, + "storage_18_linux_okhttp": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "linux", + "client": "okhttp" + }, + "storage_18_mac_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "mac", + "client": "netty" + }, + "cosmos_111_windows_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "windows", + "client": "netty" + }, + "cosmos_111_linux_okhttp": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "linux", + "client": "okhttp" + }, + "cosmos_111_mac_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. Includes/excludes from the top-level matrix get applied to the multiplied matrix, so the below element will be added + to the above matrix, for an output matrix with 7 elements: + ``` + "windows_TestFromSource_18": { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ``` + +## Matrix Generation behavior + +#### all + +`all` will output the full matrix, i.e. every possible permutation of all parameters given (p1.Length * p2.Length * ...). + +#### sparse + +`sparse` outputs the minimum number of parameter combinations while ensuring that all parameter values are present in at least one matrix job. +Effectively this means the total length of a sparse matrix will be equal to the largest matrix dimension, i.e. `max(p1.Length, p2.Length, ...)`. + +To build a sparse matrix, a full matrix is generated, and then walked diagonally N times where N is the largest matrix dimension. +This pattern works for any N-dimensional matrix, via an incrementing index (n, n, n, ...), (n+1, n+1, n+1, ...), etc. +Index lookups against matrix dimensions are calculated modulus the dimension size, so a two-dimensional matrix of 4x2 might be walked like this: + +``` +index: 0, 0: +o . . . +. . . . + +index: 1, 1: +. . . . +. o . . + +index: 2, 2 (modded to 2, 0): +. . o . +. . . . + +index: 3, 3 (modded to 3, 1): +. . . . +. . . o +``` + +#### include/exclude + +Include and exclude support additions and subtractions off the base matrix. Both include and exclude take an array of matrix values. +Typically these values will be a single entry, but they also support the cross-product matrix definition syntax of the base matrix. + +Include and exclude are parsed fully. So if a sparse matrix is called for, a sparse version of the base matrix will be generated, but +the full matrix of both include and exclude will be processed. + +Excludes are processed first, so includes can be used to add back any specific jobs to the matrix. + +#### displayNames + +In the matrix job output that azure pipelines consumes, the format is a dictionary of dictionaries. For example: + +``` +{ + "net461_macOS1015": { + "framework": "net461", + "operatingSystem": "macOS-10.15" + }, + "net50_ubuntu1804": { + "framework": "net50", + "operatingSystem": "ubuntu-18.04" + }, + "netcoreapp21_windows2019": { + "framework": "netcoreapp2.1", + "operatingSystem": "windows-2019" + }, + "UseProjectRef_net461_windows2019": { + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true", + "framework": "net461", + "operatingSystem": "windows-2019" + } +} +``` + +The top level keys are used as job names, meaning they get displayed in the azure pipelines UI when running the pipeline. + +The logic for generating display names works like this: + +- Join parameter values by "_" + a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value. + b. For each name value, strip all non-alphanumeric characters (excluding "_"). + c. If the name is greater than 100 characters, truncate it. + +#### Filters + +Filters can be passed to the matrix as an array of strings, each matching the format of =. When a matrix entry +does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify +filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example. + +Display name filters can also be passed as a single regex string that runs against the [generated display name](#displaynames) of the matrix job. +The intent of display name filters is to be defined primarily as a top level variable at template queue time in the azure pipelines UI. + +For example, the below command will filter for matrix entries with "windows" in the job display name, no matrix variable +named "ExcludedKey", a framework variable containing either "461" or "5.0", and an optional key "SupportedClouds" that, if exists, must contain "Public": + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath samples/matrix.json ` + -Selection all ` + -DisplayNameFilter ".*windows.*" ` + -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*") +``` + +#### NonSparseParameters + +Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The +NonSparseParameters argument allows for more fine-grained control of matrix generation. For example: + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath /path/to/matrix.json ` + -Selection sparse ` + -NonSparseParameters @("JavaTestVersion") +``` + +Given a matrix like below with `JavaTestVersion` marked as a non-sparse parameter: + +``` +{ + "matrix": { + "Agent": { + "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }, + "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "JavaTestVersion": [ "1.8", "1.11" ], + "AZURE_TEST_HTTP_CLIENTS": "netty", + "ArmTemplateParameters": [ "@{endpointType='storage'}", "@{endpointType='cosmos'}" ] + } +} +``` + +A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters +(3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`. + +#### Under the hood + +The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, +the below config would generate a 2x2x1x1x1 matrix (five-dimensional): + +``` +"matrix": { + "framework": [ "net461", "netcoreapp2.1" ], + "additionalTestArguments": [ "", "/p:SuperTest=true" ] + "pool": [ "ubuntu-18.04" ], + "container": [ "ubuntu-18.04" ], + "testMode": [ "Record" ] +} +``` + +The matrix is stored as a one-dimensional array, with a row-major indexing scheme (e.g. `(2, 1, 0, 1, 0)`). + +## Testing + +The matrix functions can be tested using [pester](https://pester.dev/). The test command must be run from within the tests directory. + +``` +$ cd tests +$ Invoke-Pester + +Starting discovery in 3 files. +Discovery finished in 75ms. +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 750ms (309ms|428ms) +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 867ms (250ms|608ms) +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 2.71s (725ms|1.93s) +Tests completed in 4.33s +Tests Passed: 141, Failed: 0, Skipped: 4 NotRun: 0 +``` diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 new file mode 100644 index 00000000000..07b2363765c --- /dev/null +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -0,0 +1,553 @@ +Set-StrictMode -Version "4.0" + +class MatrixConfig { + [PSCustomObject]$displayNames + [Hashtable]$displayNamesLookup + [PSCustomObject]$matrix + [System.Collections.Specialized.OrderedDictionary]$orderedMatrix + [Array]$include + [Array]$exclude +} + +$IMPORT_KEYWORD = '$IMPORT' + +function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) +{ + $name = $parameter.ToString() + + if ($displayNamesLookup.ContainsKey($parameter)) { + $name = $displayNamesLookup[$parameter] + } + + # Matrix naming restrictions: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration + $name = $name -replace "[^A-Za-z0-9_]", "" + return $name +} + +function GenerateMatrix( + [MatrixConfig]$config, + [String]$selectFromMatrixType, + [String]$displayNameFilter = ".*", + [Array]$filters = @(), + [Array]$nonSparseParameters = @() +) { + $orderedMatrix, $importedMatrix = ProcessImport $config.orderedMatrix $selectFromMatrixType + if ($selectFromMatrixType -eq "sparse") { + [Array]$matrix = GenerateSparseMatrix $orderedMatrix $config.displayNamesLookup $nonSparseParameters + } elseif ($selectFromMatrixType -eq "all") { + [Array]$matrix = GenerateFullMatrix $orderedMatrix $config.displayNamesLookup + } else { + throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" + } + + # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the + # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). + if ($importedMatrix) { + [Array]$matrix = CombineMatrices $matrix $importedMatrix + } + + if ($config.exclude) { + [Array]$matrix = ProcessExcludes $matrix $config.exclude + } + if ($config.include) { + [Array]$matrix = ProcessIncludes $config $matrix $selectFromMatrixType + } + + [Array]$matrix = FilterMatrixDisplayName $matrix $displayNameFilter + [Array]$matrix = FilterMatrix $matrix $filters + return $matrix +} + +function ProcessNonSparseParameters( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Array]$nonSparseParameters +) { + if (!$nonSparseParameters) { + return $parameters, $null + } + + $sparse = [ordered]@{} + $nonSparse = [ordered]@{} + + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Name -in $nonSparseParameters) { + $nonSparse[$param.Name] = $param.Value + } else { + $sparse[$param.Name] = $param.Value + } + } + + return $sparse, $nonSparse +} + +function FilterMatrixDisplayName([array]$matrix, [string]$filter) { + return $matrix | ForEach-Object { + if ($_.Name -match $filter) { + return $_ + } + } +} + +# Filters take the format of key=valueregex,key2=valueregex2 +function FilterMatrix([array]$matrix, [array]$filters) { + $matrix = $matrix | ForEach-Object { + if (MatchesFilters $_ $filters) { + return $_ + } + } + return $matrix +} + +function MatchesFilters([hashtable]$entry, [array]$filters) { + foreach ($filter in $filters) { + $key, $regex = ParseFilter $filter + # Default all regex checks to go against empty string when keys are missing. + # This simplifies the filter syntax/interface to be regex only. + $value = "" + if ($null -ne $entry -and $entry.parameters.Contains($key)) { + $value = $entry.parameters[$key] + } + if ($value -notmatch $regex) { + return $false + } + } + + return $true +} + +function ParseFilter([string]$filter) { + # Lazy match key in case value contains '=' + if ($filter -match "(.+?)=(.+)") { + $key = $matches[1] + $regex = $matches[2] + return $key, $regex + } else { + throw "Invalid filter: `"${filter}`", expected = format" + } +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfigFromJson([String]$jsonConfig) +{ + [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + $config.orderedMatrix = [ordered]@{} + $config.displayNamesLookup = @{} + + if ($null -ne $config.matrix) { + $config.matrix.PSObject.Properties | ForEach-Object { + $config.orderedMatrix.Add($_.Name, $_.Value) + } + } + if ($null -ne $config.displayNames) { + $config.displayNames.PSObject.Properties | ForEach-Object { + $config.displayNamesLookup.Add($_.Name, $_.Value) + } + } + $config.include = $config.include | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + $config.exclude = $config.exclude | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + + return $config +} + +function ProcessExcludes([Array]$matrix, [Array]$excludes) +{ + $deleteKey = "%DELETE%" + $exclusionMatrix = @() + + foreach ($exclusion in $excludes) { + $full = GenerateFullMatrix $exclusion + $exclusionMatrix += $full + } + + foreach ($element in $matrix) { + foreach ($exclusion in $exclusionMatrix) { + $match = MatrixElementMatch $element.parameters $exclusion.parameters + if ($match) { + $element.parameters[$deleteKey] = $true + } + } + } + + return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } +} + +function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) +{ + $inclusionMatrix = @() + foreach ($inclusion in $config.include) { + $full = GenerateFullMatrix $inclusion $config.displayNamesLookup + $inclusionMatrix += $full + } + + return $matrix + $inclusionMatrix +} + +function ProcessImport([System.Collections.Specialized.OrderedDictionary]$matrix, [String]$selection) +{ + if (!$matrix -or !$matrix.Contains($IMPORT_KEYWORD)) { + return $matrix + } + + $importPath = $matrix[$IMPORT_KEYWORD] + $matrix.Remove($IMPORT_KEYWORD) + + $matrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrix = GenerateMatrix $matrixConfig $selection + + return $matrix, $importedMatrix +} + +function CombineMatrices([Array]$matrix1, [Array]$matrix2) +{ + $combined = @() + if (!$matrix1) { + return $matrix2 + } + if (!$matrix2) { + return $matrix1 + } + + foreach ($entry1 in $matrix1) { + foreach ($entry2 in $matrix2) { + $newEntry = @{ + name = $entry1.name + parameters = CloneOrderedDictionary $entry1.parameters + } + foreach($param in $entry2.parameters.GetEnumerator()) { + if (!$newEntry.Contains($param.Name)) { + $newEntry.parameters[$param.Name] = $param.Value + } else { + Write-Warning "Skipping duplicate parameter `"$($param.Name)`" when combining matrix." + } + } + + # The maximum allowed matrix name length is 100 characters + $entry2.name = $entry2.name.TrimStart("job_") + $newEntry.name = $newEntry.name, $entry2.name -join "_" + if ($newEntry.name.Length -gt 100) { + $newEntry.name = $newEntry.name[0..99] -join "" + } + + $combined += $newEntry + } + } + + return $combined +} + +function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) +{ + if ($target.Count -eq 0) { + return $false + } + + foreach ($key in $target.Keys) { + if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { + return $false + } + } + + return $true +} + +function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { + $newDictionary = [Ordered]@{} + foreach ($element in $dictionary.GetEnumerator()) { + $newDictionary[$element.Name] = $element.Value + } + return $newDictionary +} + +function SerializePipelineMatrix([Array]$matrix) +{ + $pipelineMatrix = [Ordered]@{} + foreach ($entry in $matrix) { + $pipelineMatrix.Add($entry.name, [Ordered]@{}) + foreach ($key in $entry.parameters.Keys) { + $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) + } + } + + return @{ + compressed = $pipelineMatrix | ConvertTo-Json -Compress ; + pretty = $pipelineMatrix | ConvertTo-Json; + } +} + +function GenerateSparseMatrix( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Hashtable]$displayNamesLookup, + [Array]$nonSparseParameters = @() +) { + $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters + [Array]$dimensions = GetMatrixDimensions $parameters + [Array]$matrix = GenerateFullMatrix $parameters $displayNamesLookup + + $sparseMatrix = @() + $indexes = GetSparseMatrixIndexes $dimensions + foreach ($idx in $indexes) { + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } + + if ($nonSparse) { + [Array]$allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup + return CombineMatrices $allOfMatrix $sparseMatrix + } + + return $sparseMatrix +} + +function GetSparseMatrixIndexes([Array]$dimensions) +{ + $size = ($dimensions | Measure-Object -Maximum).Maximum + $indexes = @() + + # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. + # For example, given a matrix with dimensions 3, 2, 2: + # 0, 0, 0 + # 1, 1, 1 + # 2, 2, 2 + # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions + for ($i = 0; $i -lt $size; $i++) { + $idx = @() + for ($j = 0; $j -lt $dimensions.Length; $j++) { + $idx += $i % $dimensions[$j] + } + $indexes += ,$idx + } + + return $indexes +} + +function GenerateFullMatrix( + [System.Collections.Specialized.OrderedDictionary] $parameters, + [Hashtable]$displayNamesLookup = @{} +) { + # Handle when the config does not have a matrix specified (e.g. only the include field is specified) + if ($parameters.Count -eq 0) { + return @() + } + + $parameterArray = $parameters.GetEnumerator() | ForEach-Object { $_ } + + $matrix = [System.Collections.ArrayList]::new() + InitializeMatrix $parameterArray $displayNamesLookup $matrix + + return $matrix +} + +function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$permutation, [Hashtable]$displayNamesLookup = @{}) +{ + $names = @() + $splattedParameters = [Ordered]@{} + + foreach ($entry in $permutation.GetEnumerator()) { + $nameSegment = "" + + if ($entry.Value -is [PSCustomObject]) { + $nameSegment = CreateDisplayName $entry.Name $displayNamesLookup + foreach ($toSplat in $entry.Value.PSObject.Properties) { + $splattedParameters.Add($toSplat.Name, $toSplat.Value) + } + } else { + $nameSegment = CreateDisplayName $entry.Value $displayNamesLookup + $splattedParameters.Add($entry.Name, $entry.Value) + } + + if ($nameSegment) { + $names += $nameSegment + } + } + + # The maximum allowed matrix name length is 100 characters + $name = $names -join "_" + if ($name.Length -gt 100) { + $name = $name[0..99] -join "" + } + $stripped = $name -replace "^[^A-Za-z]*", "" # strip leading digits + if ($stripped -eq "") { + $name = "job_" + $name # Handle names that consist entirely of numbers + } else { + $name = $stripped + } + + return @{ + name = $name + parameters = $splattedParameters + } +} + +function InitializeMatrix +{ + param( + [Array]$parameters, + [Hashtable]$displayNamesLookup, + [System.Collections.ArrayList]$permutations, + $permutation = [Ordered]@{} + ) + $head, $tail = $parameters + + if (!$head) { + $entry = CreateMatrixEntry $permutation $displayNamesLookup + $permutations.Add($entry) | Out-Null + return + } + + # This behavior implicitly treats non-array values as single elements + foreach ($value in $head.Value) { + $newPermutation = CloneOrderedDictionary $permutation + if ($value -is [PSCustomObject]) { + foreach ($nestedParameter in $value.PSObject.Properties) { + $nestedPermutation = CloneOrderedDictionary $newPermutation + $nestedPermutation[$nestedParameter.Name] = $nestedParameter.Value + InitializeMatrix $tail $displayNamesLookup $permutations $nestedPermutation + } + } else { + $newPermutation[$head.Name] = $value + InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation + } + } +} + +function GetMatrixDimensions([System.Collections.Specialized.OrderedDictionary]$parameters) +{ + $dimensions = @() + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Value -is [PSCustomObject]) { + $dimensions += ($param.Value.PSObject.Properties | Measure-Object).Count + } elseif ($param.Value -is [Array]) { + $dimensions += $param.Value.Length + } else { + $dimensions += 1 + } + } + + return $dimensions +} + +function SetNdMatrixElement +{ + param( + $element, + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" + } + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + $matrix[$arrayIndex] = $element +} + +function GetNdMatrixArrayIndex +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" + } + + $stride = 1 + # Commented out does lookup with wrap handling + # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] + $index = $idx[$idx.Length-1] + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + # Commented out does lookup with wrap handling + # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride + $index += $idx[$i-1] * $stride + } + + return $index +} + +function GetNdMatrixElement +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + return $matrix[$arrayIndex] +} + +function GetNdMatrixIndex +{ + param( + [int]$index, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $matrixIndex = @() + $stride = 1 + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + $page = [math]::floor($index / $stride) % $dimensions[$i-1] + $matrixIndex = ,$page + $matrixIndex + } + $col = $index % $dimensions[$dimensions.Length-1] + $matrixIndex += $col + + return $matrixIndex +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +# The below functions are non-dynamic examples that # +# help explain the above N-dimensional algorithm # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) +{ + $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] + $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] + $stride3 = $idx[2] * $dimensions[3] + $stride4 = $idx[3] + + return $matrix[$stride1 + $stride2 + $stride3 + $stride4] +} + +function Get4dMatrixIndex([int]$index, [Array]$dimensions) +{ + $stride1 = $dimensions[3] + $stride2 = $dimensions[2] + $stride3 = $dimensions[1] + $page1 = [math]::floor($index / $stride1) % $dimensions[2] + $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] + $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] + $remainder = $index % $dimensions[3] + + return @($page3, $page2, $page1, $remainder) +} + diff --git a/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml b/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml new file mode 100644 index 00000000000..9d7c8c9a789 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml @@ -0,0 +1,42 @@ +parameters: +- name: CloudConfig + type: object +- name: Matrix + type: string +- name: DependsOn + type: string + default: '' +- name: UsePlatformContainer + type: boolean + default: false + +jobs: + - job: + dependsOn: ${{ parameters.DependsOn }} + condition: ne(${{ parameters.Matrix }}, '{}') + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + + pool: + name: $(Pool) + vmImage: $(OSVmImage) + + ${{ if eq(parameters.UsePlatformContainer, 'true') }}: + container: $[ variables['Container'] ] + + steps: + - pwsh: | + Write-Output "MATRIX JOB PARAMETERS" + Write-Output $(Agent.JobName) + Write-Output "-----------------" + Write-Output $(OSVmImage) + Write-Output $(TestTargetFramework) + try { + Write-Output $(additionalTestArguments) + } catch {} + displayName: Print matrix job variables + + - pwsh: | + Write-Output "Success" + displayName: linux OS condition example + condition: and(succeededOrFailed(), contains(variables['OSVmImage'], 'Ubuntu')) diff --git a/eng/common/scripts/job-matrix/samples/matrix-test.yml b/eng/common/scripts/job-matrix/samples/matrix-test.yml new file mode 100644 index 00000000000..be9d907e7c2 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix-test.yml @@ -0,0 +1,26 @@ +trigger: none + +variables: + - template: /eng/pipelines/templates/variables/globals.yml + +jobs: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + JobTemplatePath: /eng/common/scripts/job-matrix/samples/matrix-job-sample.yml + AdditionalParameters: [] + CloudConfig: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Location: eastus2 + Cloud: Public + MatrixFilters: [] + MatrixConfigs: + - Name: base_product_matrix + Path: eng/common/scripts/job-matrix/samples/matrix.json + Selection: all + GenerateVMJobs: true + - Name: sparse_product_matrix + Path: eng/common/scripts/job-matrix/samples/matrix.json + Selection: sparse + NonSparseParameters: + - framework + GenerateVMJobs: true diff --git a/eng/common/scripts/job-matrix/samples/matrix.json b/eng/common/scripts/job-matrix/samples/matrix.json new file mode 100644 index 00000000000..cf88ef02700 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix.json @@ -0,0 +1,28 @@ +{ + "displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" + }, + "matrix": { + "Agent": { + "ubuntu-18.04": { "OSVmImage": "ubuntu-18.04", "Pool": "Azure Pipelines" }, + "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "TestTargetFramework": [ "netcoreapp2.1", "net461", "net5.0" ] + }, + "include": [ + { + "Agent": { + "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" } + }, + "TestTargetFramework": [ "net461", "net5.0" ], + "AdditionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "OSVmImage": "MMS2019", + "framework": "netcoreapp2.1" + } + ] +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 new file mode 100644 index 00000000000..a25367bebff --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 @@ -0,0 +1,84 @@ +Import-Module Pester + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "matrix": { + "operatingSystem": [ "windows-2019", "ubuntu-18.04", "macOS-10.15" ], + "framework": [ "net461", "netcoreapp2.1" ], + "additionalArguments": [ "", "mode=test" ] + } +} +"@ + $config = GetMatrixConfigFromJson $matrixConfig +} + +Describe "Matrix Filter" -Tag "filter" { + It "Should filter by matrix display name" -TestCases @( + @{ regex = "windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ regex = "windows2019_netcoreapp21_modetest"; expectedFirst = "windows2019_netcoreapp21_modetest"; length = 1 } + @{ regex = ".*ubuntu.*"; expectedFirst = "ubuntu1804_net461"; length = 4 } + ) { + [array]$matrix = GenerateMatrix $config "all" $regex + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle no display name filter matches" { + $matrix = GenerateMatrix $config "all" + [array]$filtered = FilterMatrixDisplayName $matrix "doesnotexist" + $filtered | Should -BeNullOrEmpty + } + + It "Should filter by matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "operatingSystem=windows-2019"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "framework=.*"; expectedFirst = "windows2019_net461"; length = 12 } + @{ filterString = "additionalArguments=mode=test"; expectedFirst = "windows2019_net461_modetest"; length = 6 } + @{ filterString = "additionalArguments=^$"; expectedFirst = "windows2019_net461"; length = 6 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should filter by optional matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=^$|windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "doesnotexist=^$|.*"; expectedFirst = "windows2019_net461"; length = 12 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle multiple matrix key/value filters " { + [array]$matrix = GenerateMatrix $config "all" -filters "operatingSystem=windows.*","framework=.*","additionalArguments=mode=test" + $matrix.Length | Should -Be 2 + $matrix[0].Name | Should -Be "windows2019_net461_modetest" + } + + It "Should handle no matrix key/value filter matches" { + [array]$matrix = GenerateMatrix $config "all" -filters @("doesnot=exist") + $matrix | Should -BeNullOrEmpty + } + + It "Should handle invalid matrix key/value filter syntax" { + { GenerateMatrix $config "all" -filters @("invalid") } | Should -Throw + { GenerateMatrix $config "all" -filters @("emptyvalue=") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=emptykey") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=") } | Should -Throw + } + + It "Should filter by key exclude" { + [array]$matrix = GenerateMatrix $config "all" -filters @("operatingSystem=^$") + $matrix | Should -BeNullOrEmpty + + [array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 12 + $matrix += @{ Name = "excludeme"; Parameters = [Ordered]@{ "foo" = 1 } } + [array]$matrix = FilterMatrix $matrix @("foo=^$") + $matrix.Length | Should -Be 12 + } +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 new file mode 100644 index 00000000000..748c6632d0f --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -0,0 +1,219 @@ +Import-Module Pester + + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + function CompareMatrices([Array]$matrix, [Array]$expected) { + $matrix.Length | Should -Be $expected.Length + + for ($i = 0; $i -lt $matrix.Length; $i++) { + foreach ($entry in $matrix[$i]) { + $expected[$i].name | Should -Be $entry.name + foreach ($param in $entry.parameters.GetEnumerator()) { + $expected[$i].parameters[$param.Name] | Should -Be $param.Value + } + } + } + } +} + +Describe "Platform Matrix nonSparse" -Tag "nonsparse" { + BeforeEach { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2, 3 ], + "testField3": [ 1, 2, 3, 4 ], + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + } + + It "Should process nonSparse parameters" { + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField1","testField3" + $parameters.Count | Should -Be 1 + $parameters["testField2"] | Should -Be 1,2,3 + $nonSparse.Count | Should -Be 2 + $nonSparse["testField1"] | Should -Be 1,2 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField3" + $parameters.Count | Should -Be 2 + $parameters.Contains("testField3") | Should -Be $false + $nonSparse.Count | Should -Be 1 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + } + + It "Should ignore nonSparse with all selection" { + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 24 + } + + It "Should combine sparse matrix with nonSparse parameters" { + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 12 + } + + It "Should combine with multiple nonSparse fields" { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2 ], + "testField3": [ 31, 32 ], + "testField4": [ 41, 42 ] + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 16 + + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 8 + } +} + +Describe "Platform Matrix Import" -Tag "import" { + It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "testField" + + $matrix.Length | Should -Be 6 + + $matrix[0].name | Should -Be test1_foo1_bar1 + $matrix[0].parameters.testField | Should -Be "test1" + $matrix[0].parameters.Foo | Should -Be "foo1" + $matrix[2].name | Should -Be test1_importedBaz + $matrix[2].parameters.testField | Should -Be "test1" + $matrix[2].parameters.Baz | Should -Be "importedBaz" + $matrix[4].name | Should -Be test2_foo2_bar2 + $matrix[4].parameters.testField | Should -Be "test2" + $matrix[4].parameters.Foo | Should -Be "foo2" + } + + It "Should generate a sparse matrix with an imported a sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField1": [ "test11", "test12" ], + "testField2": [ "test21", "test22" ] + } +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo1", "Bar": "bar1" }, + "name": "test11_test21_foo1_bar1" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo2", "Bar": "bar2" }, + "name": "test11_test21_foo2_bar2" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Baz": "importedBaz" }, + "name": "test11_test21_importedBaz" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo1", "Bar": "bar1" }, + "name": "test12_test22_foo1_bar1" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo2", "Bar": "bar2" }, + "name": "test12_test22_foo2_bar2" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Baz": "importedBaz" }, + "name": "test12_test22_importedBaz" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 6 + CompareMatrices $matrix $expected + } + + It "Should import a sparse matrix with import, include, and exclude" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2", "test3" ], + }, + "include": [ + { + "testImportIncludeName": [ "testInclude1", "testInclude2" ] + } + ], + "exclude": [ + { + "testField": "test1" + }, + { + "testField": "test3", + "Baz": "importedBaz" + } + ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField": "test2", "Foo": "foo1", "Bar": "bar1" }, + "name": "test2_foo1_bar1" + }, + { + "parameters": { "testField": "test2", "Foo": "foo2", "Bar": "bar2" }, + "name": "test2_foo2_bar2" + }, + { + "parameters": { "testField": "test2", "Baz": "importedBaz" }, + "name": "test2_importedBaz" + }, + { + "parameters": { "testField": "test3", "Foo": "foo1", "Bar": "bar1" }, + "name": "test3_foo1_bar1" + }, + { + "parameters": { "testField": "test3", "Foo": "foo2", "Bar": "bar2" }, + "name": "test3_foo2_bar2" + }, + { + "parameters": { "testImportIncludeName": "testInclude1" }, + "name": "testInclude1" + }, + { + "parameters": { "testImportIncludeName": "testInclude2" }, + "name": "testInclude2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 7 + CompareMatrices $matrix $expected + } +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 new file mode 100644 index 00000000000..ab9ce82a00f --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 @@ -0,0 +1,562 @@ +Import-Module Pester + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": ["net461", "netcoreapp2.1", "net50"], + "additionalArguments": "--enableWindowsFoo" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + }, + { + "operatingSystem": "macOS-10.15", + "framework": "netcoreapp2.1" + }, + { + "operatingSystem": ["macOS-10.15", "ubuntu-18.04"], + "additionalArguments": "--enableFoo" + } + ] +} +"@ +} + +Describe "Matrix-Lookup" -Tag "lookup" { + It "Should navigate a 2d matrix: " -TestCases @( + @{ row = 0; col = 0; expected = 1 }, + @{ row = 0; col = 1; expected = 2 }, + @{ row = 1; col = 0; expected = 3 }, + @{ row = 1; col = 1; expected = 4 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 3d matrix: " -TestCases @( + @{ z = 0; row = 0; col = 0; expected = 1 } + @{ z = 0; row = 0; col = 1; expected = 2 } + @{ z = 0; row = 1; col = 0; expected = 3 } + @{ z = 0; row = 1; col = 1; expected = 4 } + @{ z = 1; row = 0; col = 0; expected = 5 } + @{ z = 1; row = 0; col = 1; expected = 6 } + @{ z = 1; row = 1; col = 0; expected = 7 } + @{ z = 1; row = 1; col = 1; expected = 8 } + ) { + $dimensions = @(2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8 + ) + GetNdMatrixElement @($z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 1; col = 0; expected = 3 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 4 } + @{ t = 0; z = 1; row = 0; col = 0; expected = 5 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 6 } + @{ t = 0; z = 1; row = 1; col = 0; expected = 7 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 8 } + @{ t = 1; z = 0; row = 0; col = 0; expected = 9 } + @{ t = 1; z = 0; row = 0; col = 1; expected = 10 } + @{ t = 1; z = 0; row = 1; col = 0; expected = 11 } + @{ t = 1; z = 0; row = 1; col = 1; expected = 12 } + @{ t = 1; z = 1; row = 0; col = 0; expected = 13 } + @{ t = 1; z = 1; row = 0; col = 1; expected = 14 } + @{ t = 1; z = 1; row = 1; col = 0; expected = 15 } + @{ t = 1; z = 1; row = 1; col = 1; expected = 16 } + ) { + $dimensions = @(2, 2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 0; col = 2; expected = 3 } + @{ t = 0; z = 0; row = 0; col = 3; expected = 4 } + + @{ t = 0; z = 0; row = 1; col = 0; expected = 5 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 6 } + @{ t = 0; z = 0; row = 1; col = 2; expected = 7 } + @{ t = 0; z = 0; row = 1; col = 3; expected = 8 } + + @{ t = 0; z = 1; row = 0; col = 0; expected = 9 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 10 } + @{ t = 0; z = 1; row = 0; col = 2; expected = 11 } + @{ t = 0; z = 1; row = 0; col = 3; expected = 12 } + + @{ t = 0; z = 1; row = 1; col = 0; expected = 13 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 14 } + @{ t = 0; z = 1; row = 1; col = 2; expected = 15 } + @{ t = 0; z = 1; row = 1; col = 3; expected = 16 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + # Skipping since by default wrapping behavior on indexing is disabled. + # Keeping here in case we want to enable it. + It -Skip "Should handle index wrapping: " -TestCases @( + @{ row = 2; col = 2; expected = 1 } + @{ row = 2; col = 3; expected = 2 } + @{ row = 4; col = 4; expected = 1 } + @{ row = 4; col = 5; expected = 2 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } +} + +Describe "Matrix-Reverse-Lookup" -Tag "lookup" { + It "Should lookup a 2d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0) } + @{ index = 1; expected = @(0,1) } + @{ index = 2; expected = @(1,0) } + @{ index = 3; expected = @(1,1) } + ) { + $dimensions = @(2, 2) + $matrix = @(1, 2, 3, 4) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + ) { + $dimensions = @(2, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,0,2) } + + @{ index = 3; expected = @(0,1,0) } + @{ index = 4; expected = @(0,1,1) } + @{ index = 5; expected = @(0,1,2) } + + @{ index = 6; expected = @(1,0,0) } + @{ index = 7; expected = @(1,0,1) } + @{ index = 8; expected = @(1,0,2) } + + @{ index = 9; expected = @(1,1,0) } + @{ index = 10; expected = @(1,1,1) } + @{ index = 11; expected = @(1,1,2) } + ) { + $dimensions = @(2, 2, 3) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + + @{ index = 8; expected = @(2,0,0) } + @{ index = 9; expected = @(2,0,1) } + @{ index = 10; expected = @(2,1,0) } + @{ index = 11; expected = @(2,1,1) } + ) { + $dimensions = @(3, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 4d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0,0) } + @{ index = 1; expected = @(0,0,0,1) } + @{ index = 2; expected = @(0,0,0,2) } + @{ index = 3; expected = @(0,0,0,3) } + + @{ index = 4; expected = @(0,0,1,0) } + @{ index = 5; expected = @(0,0,1,1) } + @{ index = 6; expected = @(0,0,1,2) } + @{ index = 7; expected = @(0,0,1,3) } + + @{ index = 8; expected = @(0,1,0,0) } + @{ index = 9; expected = @(0,1,0,1) } + @{ index = 10; expected = @(0,1,0,2) } + @{ index = 11; expected = @(0,1,0,3) } + + @{ index = 12; expected = @(0,1,1,0) } + @{ index = 13; expected = @(0,1,1,1) } + @{ index = 14; expected = @(0,1,1,2) } + @{ index = 15; expected = @(0,1,1,3) } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } +} + +Describe 'Matrix-Set' -Tag "set" { + It "Should set a matrix element" -TestCases @( + @{ value = "set"; index = @(0,0,0,0); arrayIndex = 0 } + @{ value = "ones"; index = @(0,1,1,1); arrayIndex = 13 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + + SetNdMatrixElement $value $index $matrix $dimensions + $matrix[$arrayIndex] | Should -Be $value + } +} + +Describe "Platform Matrix Generation" -Tag "generate" { + BeforeEach { + $matrixConfigForGenerate = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": "net461", + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "foo": "bar" + }, + { + "foo2": "bar2" + } + ] +} +"@ + $generateConfig = GetMatrixConfigFromJson $matrixConfigForGenerate + } + + It "Should get matrix dimensions from Nd parameters" { + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2 + + $generateConfig.orderedMatrix.Add("testStringParameter", "test") + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2, 1 + } + + It "Should use name overrides from displayNames" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_net461" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name | Should -Be "ubuntu1804_netcoreapp21_withFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.name | Should -Be "macOS1015_netcoreapp21_withFoo" + } + + It "Should enforce valid display name format" { + $generateconfig.displayNamesLookup["net461"] = '123.Some.456.Invalid_format-name$(foo)' + $generateconfig.displayNamesLookup["netcoreapp2.1"] = (New-Object string[] 150) -join "a" + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateconfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_123some456invalid_formatnamefoo" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name.Length | Should -Be 100 + # The withfoo part of the argument gets cut off at the character limit + $element.name | Should -BeLike "ubuntu1804_aaaaaaaaaaaaaaaaa*" + } + + + It "Should initialize an N-dimensional matrix from all parameter permutations" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $matrix.Count | Should -Be 12 + + $element = $matrix[0].parameters + $element.operatingSystem | Should -Be "windows-2019" + $element.framework | Should -Be "net461" + $element.additionalArguments | Should -Be "" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "ubuntu-18.04" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "macOS-10.15" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + } + + It "Should initialize a sparse matrix from an N-dimensional matrix" -TestCases @( + @{ i = 0; name = "windows2019_net461"; operatingSystem = "windows-2019"; framework = "net461"; additionalArguments = ""; } + @{ i = 1; name = "ubuntu1804_netcoreapp21_withfoo"; operatingSystem = "ubuntu-18.04"; framework = "netcoreapp2.1"; additionalArguments = "--enableFoo"; } + @{ i = 2; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } + ) { + $sparseMatrix = GenerateSparseMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $size = ($dimensions | Measure-Object -Maximum).Maximum + $sparseMatrix.Count | Should -Be $size + + $sparseMatrix[$i].name | Should -Be $name + $element = $sparseMatrix[$i].parameters + $element.operatingSystem | Should -Be $operatingSystem + $element.framework | Should -Be $framework + $element.additionalArguments | Should -Be $additionalArguments + } + + It "Should generate a sparse matrix from an N-dimensional matrix config" { + $sparseMatrix = GenerateMatrix $generateConfig "sparse" + $sparseMatrix.Length | Should -Be 4 + } + + It "Should initialize a full matrix from an N-dimensional matrix config" { + $matrix = GenerateMatrix $generateConfig "all" + $matrix.Length | Should -Be 13 + } +} + +Describe "Config File Object Conversion" -Tag "convert" { + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } + + It "Should convert a matrix config" { + $config.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + $config.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" + + $config.displayNamesLookup | Should -BeOfType [Hashtable] + $config.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" + + $config.include | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + $config.exclude | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + } +} + +Describe "Platform Matrix Post Transformation" -Tag "transform" { + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } + + It "Should match partial matrix elements" -TestCases @( + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2; c = 3 }; expected = $false } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ }; expected = $false } + @{ source = [Ordered]@{ }; target = [Ordered]@{ a = 1; b = 2; }; expected = $false } + ) { + MatrixElementMatch $source $target | Should -Be $expected + } + + It "Should remove matrix elements based on exclude filters" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 5 + + $matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + [array]$withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 1 + } + + It "Should add matrix elements based on include elements" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withInclusion = ProcessIncludes $config $matrix "all" + $withInclusion.Length | Should -Be 15 + } + + It "Should include and exclude values with a matrix" { + [Array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 8 + + $matrix[0].name | Should -Be "windows2019_netcoreapp21" + $matrix[0].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[0].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[0].parameters.additionalArguments | Should -Be "" + + $matrix[1].name | Should -Be "windows2019_netcoreapp21_withfoo" + $matrix[1].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[1].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[1].parameters.additionalArguments | Should -Be "--enableFoo" + + $matrix[2].name | Should -Be "ubuntu1804_net461" + $matrix[2].parameters.framework | Should -Be "net461" + $matrix[2].parameters.operatingSystem | Should -Be "ubuntu-18.04" + $matrix[2].parameters.additionalArguments | Should -Be "" + + $matrix[4].name | Should -Be "macOS1015_net461" + $matrix[4].parameters.framework | Should -Be "net461" + $matrix[4].parameters.operatingSystem | Should -Be "macOS-10.15" + $matrix[4].parameters.additionalArguments | Should -Be "" + + $matrix[7].name | Should -Be "windows2019_net50_enableWindowsFoo" + $matrix[7].parameters.framework | Should -Be "net50" + $matrix[7].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[7].parameters.additionalArguments | Should -Be "--enableWindowsFoo" + } +} + +Describe "Platform Matrix Generation With Object Fields" -Tag "objectfields" { + BeforeEach { + $matrixConfigForObject = @" +{ + "matrix": { + "testObject": { + "testObjectName1": { "testObject1Value1": "1", "testObject1Value2": "2" }, + "testObjectName2": { "testObject2Value1": "1", "testObject2Value2": "2" } + }, + "secondTestObject": { + "secondTestObjectName1": { "secondTestObject1Value1": "1", "secondTestObject1Value2": "2" } + }, + "testField": [ "footest", "bartest" ] + }, + "include": [ + { + "testObjectInclude": { + "testObjectIncludeName": { "testObjectValue1": "1", "testObjectValue2": "2" } + }, + "testField": "footest" + } + ] +} +"@ + $objectFieldConfig = GetMatrixConfigFromJson $matrixConfigForObject + } + + It "Should parse dimensions properly" { + [Array]$dimensions = GetMatrixDimensions $objectFieldConfig.orderedMatrix + $dimensions.Length | Should -Be 3 + $dimensions[0] | Should -Be 2 + $dimensions[1] | Should -Be 1 + $dimensions[2] | Should -Be 2 + } + + It "Should populate a sparse matrix dimensions properly" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "sparse" + $matrix.Length | Should -Be 3 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[1].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[1].parameters.testField | Should -Be "bartest" + $matrix[1].parameters.testObject2Value1 | Should -Be "1" + $matrix[1].parameters.testObject2Value2 | Should -Be "2" + $matrix[1].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[1].parameters.Count | Should -Be 5 + + $matrix[2].name | Should -Be "testObjectIncludeName_footest" + $matrix[2].parameters.testField | Should -Be "footest" + $matrix[2].parameters.testObjectValue1 | Should -Be "1" + $matrix[2].parameters.testObjectValue2 | Should -Be "2" + $matrix[2].parameters.Count | Should -Be 3 + } + + It "Should splat matrix entries that are objects into key/values" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "all" + $matrix.Length | Should -Be 5 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[3].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[3].parameters.testField | Should -Be "bartest" + $matrix[3].parameters.testObject2Value1 | Should -Be "1" + $matrix[3].parameters.testObject2Value2 | Should -Be "2" + $matrix[3].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[3].parameters.Count | Should -Be 5 + + $matrix[4].name | Should -Be "testObjectIncludeName_footest" + $matrix[4].parameters.testField | Should -Be "footest" + $matrix[4].parameters.testObjectValue1 | Should -Be "1" + $matrix[4].parameters.testObjectValue2 | Should -Be "2" + $matrix[4].parameters.Count | Should -Be 3 + } +} diff --git a/eng/common/scripts/job-matrix/tests/test-import-matrix.json b/eng/common/scripts/job-matrix/tests/test-import-matrix.json new file mode 100644 index 00000000000..2048bd390c5 --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/test-import-matrix.json @@ -0,0 +1,11 @@ +{ + "matrix": { + "Foo": [ "foo1", "foo2" ], + "Bar": [ "bar1", "bar2" ] + }, + "include": [ + { + "Baz": "importedBaz" + } + ] +}