Sync tools folder from main branch to generation branch (#19430)

Co-authored-by: azurepowershell <azurepowershell@ms.com>

.azure-pipelines/code-gen.yml

@@ -0,0 +1,95 @@
+# Variable 'BaseBranch' was defined in the Variables tab
+# Variable 'BotAccessToken' was defined in the Variables tab
+# Variable 'ServiceName' was defined in the Variables tab
+# Multi-job configuration must be converted to matrix strategy: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: refs/heads/generation
+jobs:
+- job: Job_1
+  displayName: 'Service: '
+  timeoutInMinutes: 90
+  pool:
+    name: pool-windows-2019
+  steps:
+  - checkout: self
+  - task: NodeTool@0
+    displayName: Use Node 14.15.5
+    inputs:
+      versionSpec: 14.15.5
+  - task: Npm@1
+    displayName: Install autorest
+    inputs:
+      command: custom
+      verbose: false
+      customCommand: install -g  autorest@latest
+  - task: PowerShell@2
+    displayName: Generate
+    inputs:
+      targetType: inline
+      script: >-
+        $subModuleFolders =  Get-ChildItem -Directory -Filter *.Autorest
+
+        if ($subModuleFolders -eq $null) {
+          autorest --max-memory-size=8192
+        } else {
+          $subModuleFolders | foreach-object { cd $_.FullName; autorest --max-memory-size=8192}
+        }
+      pwsh: true
+      workingDirectory: src/$(ServiceName)/
+  - task: PowerShell@2
+    displayName: Build
+    condition: eq(variables.ServiceSubmoduleName, '')
+    inputs:
+      targetType: inline
+      script: >-
+        $subModuleFolders =  Get-ChildItem -Directory -Filter *.Autorest
+
+        if ($subModuleFolders -eq $null) {
+          ./build-module.ps1
+        } else {
+          $subModuleFolders | foreach-object { cd $_.FullName; ./build-module.ps1; cd ..}
+        }
+
+        mkdir ../../artifacts/src/$env:SERVICENAME
+
+        cp -r ./* ../../artifacts/src/$env:SERVICENAME
+      pwsh: true
+      workingDirectory: src/$(ServiceName)
+  - task: CmdLine@2
+    displayName: Checkout main branch
+    inputs:
+      script: >
+        RMDIR /Q/S src\%SERVICENAME%
+        git checkout src\%SERVICENAME%
+        git checkout -b codegen/%SERVICENAME% origin/%BaseBranch%
+  - task: PowerShell@2
+    displayName: Move code to main
+    condition: eq(variables.ServiceSubmoduleName, '')
+    inputs:
+      targetType: inline
+      script: >-
+        #Uninstall-Module -Name PackageManagement -AllVersions
+        Install-Module -Name PowerShellGet -RequiredVersion 2.2.3 -Force
+        Install-Module -Name platyPS -RequiredVersion 0.14.2 -Force
+        Install-Module -Name Az.Accounts -Force
+        Import-Module .\tools\Gen2Master\MoveFromGeneration2Master.ps1
+
+        $subModuleFolders = Get-ChildItem .\artifacts\src\$env:SERVICENAME -Directory -Filter *.Autorest
+        if ($subModuleFolders -eq $null) {
+          Move-Generation2Master -SourcePath .\artifacts\src\$env:SERVICENAME -DestPath .\src\$env:SERVICENAME
+        } else {
+          Move-Generation2MasterHybrid -SourcePath .\artifacts\src\$env:SERVICENAME -DestPath .\src\$env:SERVICENAME
+        }
+
+        git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
+        git config user.name "azure-powershell-bot"
+        git add ./src
+        git add tools/CreateMappings_rules.json
+        git commit -m "Move $env:SERVICENAME to $env:BaseBranch"
+        git remote set-url origin https://azure-powershell-bot:$(BotAccessToken)@github.com/Azure/azure-powershell.git
+        git push origin codegen/$env:SERVICENAME
+      pwsh: true
+

.azure-pipelines/security-tools.yml

@@ -0,0 +1,76 @@
+# Variable 'IsGenerateBased' was defined in the Variables tab
+# Variable 'NugetSecurityAnalysisWarningLevel' was defined in the Variables tab
+# Variable 'OCTOKITPAT' was defined in the Variables tab
+# Cron Schedules have been converted using UTC Time Zone and may need to be updated for your location
+schedules:
+- cron: 0 4 * * 1,2,3,4,5
+  branches:
+    include:
+    - main
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: refs/heads/main
+jobs:
+- job: Job_1
+  displayName: Main
+  timeoutInMinutes: 120
+  pool:
+    name: pool-windows-2019
+  steps:
+  - checkout: self
+    fetchTags: false
+  - task: PowerShell@2
+    displayName: Install platyPS
+    inputs:
+      targetType: inline
+      script: Install-Module platyPS -Force -Confirm:$false -Scope CurrentUser
+      pwsh: true
+  - task: NodeTool@0
+    displayName: Install Node 14.17.1
+    condition: eq(variables.IsGenerateBased, true)
+    inputs:
+      versionSpec: 14.17.1
+  - task: PowerShell@2
+    displayName: Install autorest
+    condition: eq(variables.IsGenerateBased, true)
+    inputs:
+      targetType: inline
+      script: npm install autorest@latest;$env:NODE_OPTIONS="--max-old-space-size=65536"
+  - task: PowerShell@2
+    displayName: Build
+    inputs:
+      targetType: inline
+      script: dotnet msbuilc build.proj /t:"Build;GenerateHelp" /p:"PullRequestNumber=$(System.PullRequest.PullRequestNumber);IsSecurityCheck=true"
+    env:
+        OCTOKITPAT: $(OCTOKITPAT)
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2
+    displayName: Run CredScan
+    condition: in(variables['system.pullRequest.targetBranch'], 'generation', 'main')
+    inputs:
+      toolMajorVersion: V2
+      outputFormat: sarif
+      scanFolder: SecurityTmp
+      suppressionsFile: tools/SecurityTools/CredScanSuppressions.json
+      debugMode: false
+      folderSuppression: false
+  - task: PowerShell@2
+    displayName: Cleanup Build
+    inputs:
+      targetType: inline
+      script: ./tools/CleanupBuild.ps1
+      pwsh: true
+  - task: PoliCheck@1
+    displayName: Run PoliCheck
+    inputs:
+      targetArgument: $(Build.SourcesDirectory)/artifacts/Debug
+      result: $(Build.SourcesDirectory)/artifacts/result/PoliCheck.xml
+      optionsFTPATH: tools/SecurityTools/PoliCheckFileExtensions.xml
+  - task: PublishPipelineArtifact@0
+    displayName: Save artifacts
+    condition: succeededOrFailed()
+    inputs:
+      artifactName: artifacts
+      targetPath: artifacts
+...

.azure-pipelines/util/analyze-steps.yml

@@ -49,13 +49,17 @@ steps:
     command: custom
     custom: msbuild
     arguments: 'build.proj /t:GenerateHelp /p:Configuration=${{ parameters.configuration }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
+  env:
+      OCTOKITPAT: $(OCTOKITPAT)
 
 - task: DotNetCoreCLI@2
   displayName: 'Static Analysis'
   inputs:
     command: custom
     custom: msbuild
     arguments: 'build.proj /t:StaticAnalysis /p:Configuration=${{ parameters.configuration }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
+  env:
+      OCTOKITPAT: $(OCTOKITPAT)
 
 - template: publish-artifacts-steps.yml
   parameters:

.azure-pipelines/util/build-steps.yml

@@ -49,6 +49,8 @@ steps:
     command: custom
     custom: msbuild
     arguments: 'build.proj /t:Build /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
+  env:
+      OCTOKITPAT: $(OCTOKITPAT)
 
 - task: PowerShell@2
   displayName: Build-AzPredictor

.azure-pipelines/util/test-steps.yml

@@ -36,6 +36,8 @@ steps:
     command: custom
     custom: msbuild
     arguments: 'build.proj /t:${{ parameters.testTarget }} /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
+  env:
+      OCTOKITPAT: $(OCTOKITPAT)
 
 - powershell: |
    Install-Module -Name Pester -RequiredVersion 4.10.1 -Force -SkipPublisherCheck

src/Az.autorest.hybrid.props

@@ -66,7 +66,6 @@
 
   <!-- Assets -->
   <ItemGroup>
-    <PackageReference Include="PowerShellStandard.Library" Version="5.1.0" />
     <PackageReference Include="Microsoft.CSharp" Version="4.4.1" />
   </ItemGroup>
 

tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilesChangedTask.cs

@@ -100,12 +100,8 @@ public override bool Execute()
                 try
                 {
                     //The variable is set in pipeline: "azure-powershell - powershell-core"
-                    var token = Environment.GetEnvironmentVariable("NOSCOPEPAT_ADXSDKPS");
                     var client = new GitHubClient(new ProductHeaderValue("Azure"));
-                    if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && !string.IsNullOrEmpty(token))
-                    {
-                        client.Credentials = new Credentials(token);
-                    }
+                    client.Credentials = new Credentials(Environment.GetEnvironmentVariable("OCTOKITPAT"));
                     var files = client.PullRequest.Files(RepositoryOwner, RepositoryName, int.Parse(PullRequestNumber))
                                     .ConfigureAwait(false).GetAwaiter().GetResult();
                     if (files == null)

tools/RunVersionController.ps1

@@ -21,7 +21,7 @@ Param(
     [string]$GalleryName = "PSGallery",
 
     [Parameter()]
-    [switch]$SkipAzInstall
+    [string]$ArtifactsOutputPath = "$PSScriptRoot/../artifacts/Release/"
 )
 
 enum PSVersion
@@ -193,7 +193,7 @@ function Get-ExistSerializedCmdletJsonFile
 function Bump-AzVersion
 {
     Write-Host "Getting local Az information..." -ForegroundColor Yellow
-    $localAz = Test-ModuleManifest -Path "$PSScriptRoot\Az\Az.psd1"
+    $localAz = Import-PowerShellDataFile -Path "$PSScriptRoot\Az\Az.psd1"
 
     Write-Host "Getting gallery Az information..." -ForegroundColor Yellow
     $galleryAz = Find-Module -Name Az -Repository $GalleryName
@@ -202,15 +202,15 @@ function Bump-AzVersion
     $updatedModules = @()
     foreach ($localDependency in $localAz.RequiredModules)
     {
-        $galleryDependency = $galleryAz.Dependencies | where { $_.Name -eq $localDependency.Name }
-        if ($galleryDependency -eq $null)
+        $galleryDependency = $galleryAz.Dependencies | where { $_.Name -eq $localDependency.ModuleName }
+        if ($null -eq $galleryDependency)
         {
-            $updatedModules += $localDependency.Name
+            $updatedModules += $localDependency.ModuleName
             if ($versionBump -ne [PSVersion]::MAJOR)
             {
                 $versionBump = [PSVersion]::MINOR
             }
-            Write-Host "Found new added module $($localDependency.Name)"
+            Write-Host "Found new added module $($localDependency.ModuleName)"
             continue
         }
 
@@ -219,12 +219,19 @@ function Bump-AzVersion
         {
             $galleryVersion = $galleryDependency.MinimumVersion
         }
-        $localVersion = $localDependency.Version.ToString()
+
+        $localVersion = $localDependency.RequiredVersion
+        # Az.Accounts uses ModuleVersion to annote Version
+        if ([string]::IsNullOrEmpty($localVersion))
+        {
+            $localVersion = $localDependency.ModuleVersion
+        }
+
         if ($galleryVersion.ToString() -ne $localVersion)
         {
-            $updatedModules += $galleryDependency.Name
+            $updatedModules += $localDependency.ModuleName
             $currBump = Get-VersionBump -GalleryVersion $galleryVersion.ToString() -LocalVersion $localVersion
-            Write-Host "Found $currBump version bump for $($localDependency.NAME)"
+            Write-Host "Found $currBump version bump for $($localDependency.ModuleName)"
             if ($currBump -eq [PSVersion]::MAJOR)
             {
                 $versionBump = [PSVersion]::MAJOR
@@ -246,7 +253,7 @@ function Bump-AzVersion
         return
     }
 
-    $newVersion = Get-BumpedVersion -Version $localAz.Version -VersionBump $versionBump
+    $newVersion = Get-BumpedVersion -Version $localAz.ModuleVersion -VersionBump $versionBump
 
     Write-Host "New version of Az: $newVersion" -ForegroundColor Green
 
@@ -271,13 +278,26 @@ function Bump-AzVersion
         $changeLog += "#### $updatedModule"
         $changeLog += $(Get-ReleaseNotes -Module $updatedModule -RootPath $rootPath) + "`n"
     }
-
+
+    $resolvedArtifactsOutputPath = (Resolve-Path $ArtifactsOutputPath).Path
+    if(!(Test-Path $resolvedArtifactsOutputPath))
+    {
+        throw "Please check artifacts output path: $resolvedArtifactsOutputPath whether exists."
+    }
+
+    # Update-ModuleManifest requires all required modules in Az.psd1 installed in local
+    # Add artifacts as PSModulePath to skip installation
+    if(!($env:PSModulePath.Split(";").Contains($resolvedArtifactsOutputPath)))
+    {
+        $env:PSModulePath += ";$resolvedArtifactsOutputPath"
+    }
+
     Update-ModuleManifest -Path "$PSScriptRoot\Az\Az.psd1" -ModuleVersion $newVersion -ReleaseNotes $releaseNotes
     Update-ChangeLog -Content $changeLog -RootPath $rootPath
     return $versionBump
 }
 
-function Generate-AzPreview
+function Update-AzPreview
 {
     # The version of AzPrview aligns with Az
     $AzPrviewVersion = (Import-PowerShellDataFile "$PSScriptRoot\Az\Az.psd1").ModuleVersion
@@ -407,18 +427,7 @@ switch ($PSCmdlet.ParameterSetName)
                 Write-Host "Module ${ModuleName} is not GA yet. The json file: ${JsonFile} is for reference"
             }
         }
-        try
-        {
-            if(!$SkipAzInstall.IsPresent)
-            {
-                Install-Module Az -Repository $GalleryName -Force -AllowClobber
-            }
-        }
-        catch
-        {
-            throw "Please rerun in Administrator mode."
-        }
-
+
         Write-Host executing dotnet $PSScriptRoot/../artifacts/VersionController/VersionController.Netcore.dll
         dotnet $PSScriptRoot/../artifacts/VersionController/VersionController.Netcore.dll
 
@@ -435,9 +444,9 @@ switch ($PSCmdlet.ParameterSetName)
 
 # Each release needs to update AzPreview.psd1 and dotnet csv
 # Refresh AzPreview.psd1
-Generate-AzPreview
+Update-AzPreview
 
 New-CommandMappingFile
 
 # Generate dotnet csv
-&$PSScriptRoot/Docs/GenerateDotNetCsv.ps1 -FeedPsd1FullPath "$PSScriptRoot\AzPreview\AzPreview.psd1" -CustomSource "https://www.powershellgallery.com/api/v2/"
+&$PSScriptRoot/Docs/GenerateDotNetCsv.ps1 -FeedPsd1FullPath "$PSScriptRoot\AzPreview\AzPreview.psd1" -CustomSource "https://www.powershellgallery.com/api/v2/"