GH Action to populate build stage yaml files (stable only for now)

.github/workflows/updateBuildYamls.yml

@@ -0,0 +1,124 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+name: Update Channel Based ReleaseStage Yaml Files
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      stableVersion:
+        description: 'stable release version (i.e: 7.4.1)'
+        required: false
+      previewVersion:
+        description: 'preview release version (i.e: 7.4.0-preview.5)'
+        required: false
+      ltsVersion:
+        description: 'lts release version (i.e: 7.2.12)'
+        required: false
+
+defaults:
+  run:
+    shell: pwsh
+jobs:
+  update-yamls:
+    name: Update ReleaseStageYaml for Stable Channel
+    timeout-minutes: 15
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Validate Version Syntax
+        id: validate-version
+        run: |
+          $stableValue = '${{ inputs.stableVersion }}'
+          if (!($stableValue -eq ""))
+          {
+            if ($stableValue -notmatch '\d+\.\d+\.\d+$') {
+              throw "stable release tag is not for a stable build: '$stableValue'"
+            }
+          }
+
+          $previewValue = '${{ inputs.previewVersion }}'
+          if (!($previewValue -eq ""))
+          {
+            if ($previewValue -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') {
+              throw "preview release tag is not for a preview build: '$previewValue'"
+            }
+          }
+
+          $ltsValue = '${{ inputs.ltsVersion }}'
+          if (!($ltsValue -eq ""))
+          {
+            if ($ltsValue -notmatch '\d+\.\d+\.\d+$') {
+              throw "lts release tag is not for a lts build: '$ltsValue'"
+            }
+          }
+      - name: Update ReleaseStageYaml file for Stable
+        run: |
+          $stableValue = '${{ inputs.stableVersion }}'
+          $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools'
+          $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper'
+          $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1'
+          Import-Module $buildHelperModulePath
+          if (!($stableValue -eq ""))
+          {
+            Write-Verbose -Verbose "using stable version: $stableValue"
+            ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion $stableValue -Verbose -Acr All -OsFilter All
+          }
+          else
+          {
+            # Use the version from channels.json when no version is provided
+            ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All
+          }
+      - name: Update ReleaseStageYaml file for Preview
+        run: |
+          $previewValue = '${{ inputs.previewVersion }}'
+          $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools'
+          $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper'
+          $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1'
+          Import-Module $buildHelperModulePath
+          if (!($previewValue -eq ""))
+          {
+            Write-Verbose -Verbose "using preview version: $previewValue"
+            ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $previewValue -Verbose -Acr All -OsFilter All
+          }
+          else
+          {
+            ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All
+          }
+      - name: Update ReleaseStageYaml file for LTS
+        run: |
+          $ltsValue = '${{ inputs.ltsVersion }}'
+          $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools'
+          $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper'
+          $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1'
+          Import-Module $buildHelperModulePath
+          if (!($ltsValue -eq ""))
+          {
+            Write-Verbose -Verbose "using lts version: $ltsValue"
+            ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $ltsValue -Verbose -Acr All -OsFilter All
+          }
+          else
+          {
+            # Use the version from channels.json when no version is provided
+            ./build.ps1 -UpdateBuildYaml -Channel lts -Verbose -Acr All -OsFilter All
+          }
+      - name: Create Pull Request
+        if: github.event_name == 'workflow_dispatch'
+        id: cpr
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: "${{ secrets.PR_PAT }}"
+          commit-message: "Update the stableReleaseStage yaml file"
+          committer: PwshBot <pwshbot@microsoft.com>
+          author: PwshBot <pwshbot@microsoft.com>
+          title: "Update the stableReleaseStage json"
+          base: master
+          draft: false
+          branch: update-build-yaml-files
+          push-to-fork: pwshBot/PowerShell-Docker

build.ps1

@@ -34,6 +34,10 @@ param(
     [switch]
     $GenerateMatrixJson,
 
+    [Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")]
+    [switch]
+    $UpdateBuildYaml,
+
     [Parameter(ParameterSetName="GenerateMatrixJson")]
     [switch]
     $FullJson,
@@ -108,27 +112,31 @@ param(
     $Version,
 
     [Parameter(ParameterSetName="GenerateMatrixJson")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateManifestLists")]
     [Parameter(ParameterSetName="GenerateTagsYaml")]
     [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
     [string]
     $StableVersion,
 
     [Parameter(ParameterSetName="GenerateMatrixJson")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateManifestLists")]
     [Parameter(ParameterSetName="GenerateTagsYaml")]
     [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
     [string]
     $LtsVersion,
 
     [Parameter(ParameterSetName="GenerateManifestLists")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateMatrixJson")]
     [Parameter(ParameterSetName="GenerateTagsYaml")]
     [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
     [string]
     $PreviewVersion,
 
     [Parameter(ParameterSetName="GenerateManifestLists")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateMatrixJson")]
     [Parameter(ParameterSetName="GenerateTagsYaml")]
     [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
@@ -147,12 +155,14 @@ param(
     $ForcePesterInstall,
 
     [Parameter(ParameterSetName="GenerateManifestLists")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateMatrixJson")]
     [string]
     [ValidateSet('All','OnlyAcr','NoAcr')]
     $Acr,
 
     [Parameter(ParameterSetName="GenerateManifestLists")]
+    [Parameter(ParameterSetName="UpdateBuildYaml")]
     [Parameter(ParameterSetName="GenerateMatrixJson")]
     [string]
     [ValidateSet('All','Linux','Windows')]
@@ -203,6 +213,7 @@ DynamicParam {
     Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes
     Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes
     Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false
+    Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false
     Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes
     Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes
 
@@ -227,7 +238,7 @@ Begin {
 
     $ENV:DOCKER_BUILDKIT = 1
 
-    if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
+    if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
     {
         throw "Multiple Channels are not supported in this parameter set"
     }
@@ -404,7 +415,7 @@ End {
                     }
                 }
             }
-            elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) {
+            elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) {
                 if($Acr -eq 'OnlyAcr' -and !$useAcr)
                 {
                     continue
@@ -437,7 +448,7 @@ End {
 
                 $osVersion = $allMeta.meta.osVersion
                 $manifestLists = $allMeta.meta.ManifestLists
-                if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent)
+                if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent)
                 {
                     if ($osVersion) {
                         $osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag)
@@ -525,8 +536,8 @@ End {
         Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams
     }
 
-    Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
-    if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
+    Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
+    if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
         $dockerImagesToScan = ''
         # print local image names
         # used only with the -Build
@@ -665,6 +676,47 @@ End {
         }
     }
 
+    $channelsUsed = @{}
+    if ($UpdateBuildYaml.IsPresent) {
+        foreach ($repo in $tagGroups.Keys | Sort-Object) {
+            $channelGroups = $tagGroups.$repo | Group-Object -Property Channel
+            foreach($channelGroup in $channelGroups)
+            {
+                $channelName = $channelGroup.Name
+                $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci'
+                $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml"
+
+                if (!$channelsUsed.Contains($channelName))
+                {
+                    # Note: channelGroup contains entry for a channels' regular and channel's test-deps images.
+                    # But, we only want 1 <channel>ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel.
+                    $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath
+                    if ($channelReleaseStageFileExists)
+                    {
+                        Remove-Item -Path $channelReleaseStagePath
+                    }
+
+                    New-Item -Type File -Path $channelReleaseStagePath
+
+                    Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath
+                    $channelsUsed.Add($channelName, $channelGroup.Values)
+                }
+
+                $osGroups = $channelGroup.Group | Group-Object -Property os
+                foreach ($osGroup in $osGroups) {
+                    $architectureGroups = $osGroup.Group | Group-Object -Property Architecture
+                    foreach ($architectureGroup in $architectureGroups) {
+                        # Note: For each image to be built (including test-deps images) the <channel>ReleaseStage.yml file needs to have a template call for the image.
+                        foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) {
+                            Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)"
+                            Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     if ($GenerateManifestLists.IsPresent) {
         $manifestLists = @()
         $tags = @()

tools/buildHelper/buildHelper.psm1

@@ -1030,3 +1030,89 @@ Function ConvertTo-SortedDictionary {
     }
     return $sortedDictionary
 }
+
+function Get-StartOfYamlPopulated {
+    param(
+        [string]
+        $Channel,
+
+        [string]
+        $YamlFilePath
+    )
+
+    if (!$YamlFilePath)
+    {
+        throw "Yaml file $YamlFilePath provided as parameter cannot be found."
+    }
+
+    $doubleSpace = " "*2
+
+    Add-Content -Path $YamlFilePath -Value "parameters:"
+    Add-Content -Path $YamlFilePath -Value "- name: channel"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'"
+    Add-Content -Path $YamlFilePath -Value "- name: channelPath"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''"
+    Add-Content -Path $YamlFilePath -Value "- name: vmImage"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure"
+    Add-Content -Path $YamlFilePath -Value "stages:"
+    Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel"
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:"
+}
+
+function Get-TemplatePopulatedYaml {
+    param(
+        [string]
+        $YamlFilePath,
+
+        [psobject]
+        $ImageInfo
+    )
+
+    if (!$YamlFilePath)
+    {
+        throw "Yaml file $YamlFilePath provided as parameter cannot be found."
+    }
+
+    $doubleSpace = " "*2
+    $fourSpace = " "*4
+    $sixSpace = " "*6
+
+    $imageName = $ImageInfo.Name
+    $artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_")
+    $architecture = $ImageInfo.Architecture
+    $poolOS = $ImageInfo.IsLinux ? "linux" : "windows"
+    $archBasedJobName = "Build_$($poolOS)_$($architecture)"
+
+    if ($architecture -eq "arm32")
+    {
+        $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32
+    }
+
+    Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self"
+    Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:"
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param)
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps 
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'"
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'"
+    if ($poolOS -eq "linux")
+    {
+        # only need to specify buildKitValue=1 for linux
+        Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1"
+    }
+    else
+    {
+        if ($imageName -contains "2022")
+        {
+            Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'"
+            Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'"
+        }
+
+        Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3"
+    }
+
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}"
+    Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}"
+}