From 7b2510cfe7fb925ecbd7037d0bdd6dd89d935dc1 Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Mon, 24 Nov 2025 13:09:33 -0800 Subject: [PATCH] Update common Docker engineering infrastructure with latest --- eng/docker-tools/Dockerfile.WithRepo | 6 + eng/docker-tools/Dockerfile.syft | 16 + eng/docker-tools/Get-BaseImageStatus.ps1 | 33 ++ eng/docker-tools/Get-ImageBuilder.ps1 | 13 + eng/docker-tools/Get-ImageNameVars.ps1 | 12 + eng/docker-tools/Install-DotNetSdk.ps1 | 64 ++++ eng/docker-tools/Invoke-CleanupDocker.ps1 | 20 + eng/docker-tools/Invoke-ImageBuilder.ps1 | 105 ++++++ eng/docker-tools/Invoke-WithRetry.ps1 | 41 +++ eng/docker-tools/Pull-Image.ps1 | 18 + eng/docker-tools/Retain-Build.ps1 | 43 +++ eng/docker-tools/build.ps1 | 76 ++++ eng/docker-tools/readme.md | 28 ++ eng/docker-tools/templates/1es-official.yml | 70 ++++ eng/docker-tools/templates/1es-unofficial.yml | 79 ++++ eng/docker-tools/templates/1es.yml | 75 ++++ .../templates/jobs/build-images.yml | 131 +++++++ .../templates/jobs/cg-build-projects.yml | 58 +++ .../jobs/copy-base-images-staging.yml | 29 ++ .../templates/jobs/copy-base-images.yml | 39 ++ .../templates/jobs/generate-matrix.yml | 72 ++++ .../templates/jobs/post-build.yml | 87 +++++ eng/docker-tools/templates/jobs/publish.yml | 312 ++++++++++++++++ .../jobs/test-images-linux-client.yml | 30 ++ .../jobs/test-images-windows-client.yml | 25 ++ .../templates/stages/build-and-test.yml | 343 ++++++++++++++++++ .../stages/dotnet/build-and-test.yml | 133 +++++++ .../stages/dotnet/build-test-publish-repo.yml | 74 ++++ .../stages/dotnet/publish-config-nonprod.yml | 104 ++++++ .../stages/dotnet/publish-config-prod.yml | 104 ++++++ .../templates/stages/dotnet/publish.yml | 56 +++ eng/docker-tools/templates/stages/publish.yml | 80 ++++ .../stages/setup-service-connections.yml | 38 ++ .../templates/steps/annotate-eol-digests.yml | 48 +++ .../templates/steps/clean-acr-images.yml | 33 ++ .../templates/steps/cleanup-docker-linux.yml | 15 + .../steps/cleanup-docker-windows.yml | 18 + .../templates/steps/copy-base-images.yml | 52 +++ .../steps/download-build-artifact.yml | 40 ++ .../templates/steps/init-common.yml | 9 + .../templates/steps/init-docker-linux.yml | 90 +++++ .../templates/steps/init-docker-windows.yml | 48 +++ .../steps/init-matrix-build-publish.yml | 78 ++++ .../templates/steps/parse-test-arg-arrays.yml | 15 + .../templates/steps/publish-artifact.yml | 28 ++ .../templates/steps/publish-readmes.yml | 29 ++ .../templates/steps/retain-build.yml | 9 + .../templates/steps/run-imagebuilder.yml | 76 ++++ .../templates/steps/run-pwsh-with-auth.yml | 39 ++ .../templates/steps/set-dry-run.yml | 30 ++ .../steps/set-image-info-path-var.yml | 19 + .../steps/test-images-linux-client.yml | 107 ++++++ .../steps/test-images-windows-client.yml | 66 ++++ .../templates/steps/validate-branch.yml | 52 +++ .../steps/wait-for-mcr-doc-ingestion.yml | 21 ++ .../steps/wait-for-mcr-image-ingestion.yml | 37 ++ .../templates/task-prefix-decorator.yml | 63 ++++ .../templates/variables/common-paths.yml | 5 + .../templates/variables/common.yml | 82 +++++ .../templates/variables/docker-images.yml | 7 + .../variables/dotnet/build-test-publish.yml | 48 +++ .../templates/variables/dotnet/common.yml | 63 ++++ .../variables/dotnet/secrets-unofficial.yml | 5 + .../templates/variables/dotnet/secrets.yml | 17 + 64 files changed, 3663 insertions(+) create mode 100644 eng/docker-tools/Dockerfile.WithRepo create mode 100644 eng/docker-tools/Dockerfile.syft create mode 100644 eng/docker-tools/Get-BaseImageStatus.ps1 create mode 100644 eng/docker-tools/Get-ImageBuilder.ps1 create mode 100644 eng/docker-tools/Get-ImageNameVars.ps1 create mode 100644 eng/docker-tools/Install-DotNetSdk.ps1 create mode 100644 eng/docker-tools/Invoke-CleanupDocker.ps1 create mode 100644 eng/docker-tools/Invoke-ImageBuilder.ps1 create mode 100644 eng/docker-tools/Invoke-WithRetry.ps1 create mode 100644 eng/docker-tools/Pull-Image.ps1 create mode 100644 eng/docker-tools/Retain-Build.ps1 create mode 100644 eng/docker-tools/build.ps1 create mode 100644 eng/docker-tools/readme.md create mode 100644 eng/docker-tools/templates/1es-official.yml create mode 100644 eng/docker-tools/templates/1es-unofficial.yml create mode 100644 eng/docker-tools/templates/1es.yml create mode 100644 eng/docker-tools/templates/jobs/build-images.yml create mode 100644 eng/docker-tools/templates/jobs/cg-build-projects.yml create mode 100644 eng/docker-tools/templates/jobs/copy-base-images-staging.yml create mode 100644 eng/docker-tools/templates/jobs/copy-base-images.yml create mode 100644 eng/docker-tools/templates/jobs/generate-matrix.yml create mode 100644 eng/docker-tools/templates/jobs/post-build.yml create mode 100644 eng/docker-tools/templates/jobs/publish.yml create mode 100644 eng/docker-tools/templates/jobs/test-images-linux-client.yml create mode 100644 eng/docker-tools/templates/jobs/test-images-windows-client.yml create mode 100644 eng/docker-tools/templates/stages/build-and-test.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/build-and-test.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml create mode 100644 eng/docker-tools/templates/stages/dotnet/publish.yml create mode 100644 eng/docker-tools/templates/stages/publish.yml create mode 100644 eng/docker-tools/templates/stages/setup-service-connections.yml create mode 100644 eng/docker-tools/templates/steps/annotate-eol-digests.yml create mode 100644 eng/docker-tools/templates/steps/clean-acr-images.yml create mode 100644 eng/docker-tools/templates/steps/cleanup-docker-linux.yml create mode 100644 eng/docker-tools/templates/steps/cleanup-docker-windows.yml create mode 100644 eng/docker-tools/templates/steps/copy-base-images.yml create mode 100644 eng/docker-tools/templates/steps/download-build-artifact.yml create mode 100644 eng/docker-tools/templates/steps/init-common.yml create mode 100644 eng/docker-tools/templates/steps/init-docker-linux.yml create mode 100644 eng/docker-tools/templates/steps/init-docker-windows.yml create mode 100644 eng/docker-tools/templates/steps/init-matrix-build-publish.yml create mode 100644 eng/docker-tools/templates/steps/parse-test-arg-arrays.yml create mode 100644 eng/docker-tools/templates/steps/publish-artifact.yml create mode 100644 eng/docker-tools/templates/steps/publish-readmes.yml create mode 100644 eng/docker-tools/templates/steps/retain-build.yml create mode 100644 eng/docker-tools/templates/steps/run-imagebuilder.yml create mode 100644 eng/docker-tools/templates/steps/run-pwsh-with-auth.yml create mode 100644 eng/docker-tools/templates/steps/set-dry-run.yml create mode 100644 eng/docker-tools/templates/steps/set-image-info-path-var.yml create mode 100644 eng/docker-tools/templates/steps/test-images-linux-client.yml create mode 100644 eng/docker-tools/templates/steps/test-images-windows-client.yml create mode 100644 eng/docker-tools/templates/steps/validate-branch.yml create mode 100644 eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml create mode 100644 eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml create mode 100644 eng/docker-tools/templates/task-prefix-decorator.yml create mode 100644 eng/docker-tools/templates/variables/common-paths.yml create mode 100644 eng/docker-tools/templates/variables/common.yml create mode 100644 eng/docker-tools/templates/variables/docker-images.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/build-test-publish.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/common.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml create mode 100644 eng/docker-tools/templates/variables/dotnet/secrets.yml diff --git a/eng/docker-tools/Dockerfile.WithRepo b/eng/docker-tools/Dockerfile.WithRepo new file mode 100644 index 000000000..d1126b3d1 --- /dev/null +++ b/eng/docker-tools/Dockerfile.WithRepo @@ -0,0 +1,6 @@ +# Use this Dockerfile to create an ImageBuilder image +ARG IMAGE +FROM $IMAGE + +WORKDIR /repo +COPY . . diff --git a/eng/docker-tools/Dockerfile.syft b/eng/docker-tools/Dockerfile.syft new file mode 100644 index 000000000..2e564e2ae --- /dev/null +++ b/eng/docker-tools/Dockerfile.syft @@ -0,0 +1,16 @@ +ARG SYFT_IMAGE_NAME +ARG TARGET_IMAGE_NAME + +FROM ${SYFT_IMAGE_NAME} AS syft +FROM ${TARGET_IMAGE_NAME} AS scan-image + +FROM syft AS run-scan +ARG TARGET_IMAGE_NAME +ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ + SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} +USER root +RUN --mount=from=scan-image,source=/,target=/rootfs \ + ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] + +FROM scratch AS output +COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json diff --git a/eng/docker-tools/Get-BaseImageStatus.ps1 b/eng/docker-tools/Get-BaseImageStatus.ps1 new file mode 100644 index 000000000..6b2546265 --- /dev/null +++ b/eng/docker-tools/Get-BaseImageStatus.ps1 @@ -0,0 +1,33 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Outputs the status of external base images referenced in the Dockerfiles. +#> +[cmdletbinding()] +param( + # Path to the manifest file to use + [string] + $Manifest = "manifest.json", + + # Architecture to filter Dockerfiles to + [string] + $Architecture = "*", + + # A value indicating whether to run the script continously + [switch] + $Continuous, + + # Number of seconds to wait between each iteration + [int] + $ContinuousDelay = 10 +) + +Set-StrictMode -Version Latest + +$imageBuilderArgs = "getBaseImageStatus --manifest $Manifest --architecture $Architecture" +if ($Continuous) { + $imageBuilderArgs += " --continuous --continuous-delay $ContinuousDelay" +} + +& "$PSScriptRoot/Invoke-ImageBuilder.ps1" -ImageBuilderArgs $imageBuilderArgs diff --git a/eng/docker-tools/Get-ImageBuilder.ps1 b/eng/docker-tools/Get-ImageBuilder.ps1 new file mode 100644 index 000000000..3eb35240b --- /dev/null +++ b/eng/docker-tools/Get-ImageBuilder.ps1 @@ -0,0 +1,13 @@ +#!/usr/bin/env pwsh + +# Load common image names +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 +foreach ($varName in $imageNameVars.Keys) { + Set-Variable -Name $varName -Value $imageNameVars[$varName] -Scope Global +} + +& docker inspect ${imageNames.imagebuilderName} | Out-Null +if (-not $?) { + Write-Output "Pulling" + & $PSScriptRoot/Invoke-WithRetry.ps1 "docker pull ${imageNames.imagebuilderName}" +} diff --git a/eng/docker-tools/Get-ImageNameVars.ps1 b/eng/docker-tools/Get-ImageNameVars.ps1 new file mode 100644 index 000000000..0b255c925 --- /dev/null +++ b/eng/docker-tools/Get-ImageNameVars.ps1 @@ -0,0 +1,12 @@ +# Returns a hashtable of variable name-to-value mapping representing the image name variables +# used by the common build infrastructure. + +$vars = @{} +Get-Content $PSScriptRoot/templates/variables/docker-images.yml | + Where-Object { $_.Trim().Length -gt 0 -and $_.Trim() -notlike 'variables:' -and $_.Trim() -notlike '# *' } | + ForEach-Object { + $parts = $_.Split(':', 2) + $vars[$parts[0].Trim()] = $parts[1].Trim() + } + +return $vars diff --git a/eng/docker-tools/Install-DotNetSdk.ps1 b/eng/docker-tools/Install-DotNetSdk.ps1 new file mode 100644 index 000000000..114faa260 --- /dev/null +++ b/eng/docker-tools/Install-DotNetSdk.ps1 @@ -0,0 +1,64 @@ +#!/usr/bin/env pwsh +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +<# +.SYNOPSIS +Install the .NET Core SDK at the specified path. + +.PARAMETER InstallPath +The path where the .NET Core SDK is to be installed. + +.PARAMETER Channel +The version of the .NET Core SDK to be installed. + +#> +[cmdletbinding()] +param( + [string] + $InstallPath, + [string] + $Channel = "9.0" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (!(Test-Path "$InstallPath")) { + mkdir "$InstallPath" | Out-Null +} + +$IsRunningOnUnix = $PSVersionTable.contains("Platform") -and $PSVersionTable.Platform -eq "Unix" +if ($IsRunningOnUnix) { + $DotnetInstallScript = "dotnet-install.sh" +} +else { + $DotnetInstallScript = "dotnet-install.ps1" +} + +$DotnetInstallScriptPath = Join-Path -Path $InstallPath -ChildPath $DotnetInstallScript + +if (!(Test-Path $DotnetInstallScriptPath)) { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; + & "$PSScriptRoot/Invoke-WithRetry.ps1" "Invoke-WebRequest 'https://builds.dotnet.microsoft.com/dotnet/scripts/v1/$DotnetInstallScript' -OutFile $DotnetInstallScriptPath" +} + +$DotnetChannel = $Channel + +$InstallFailed = $false +if ($IsRunningOnUnix) { + & chmod +x $DotnetInstallScriptPath + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath --channel $DotnetChannel --install-dir $InstallPath" -Retries 5 + $InstallFailed = ($LASTEXITCODE -ne 0) +} +else { + & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath -Channel $DotnetChannel -InstallDir $InstallPath" -Retries 5 + $InstallFailed = (-not $?) +} + +# See https://github.com/NuGet/NuGet.Client/pull/4259 +$Env:NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY = "6,1500" + +if ($InstallFailed) { throw "Failed to install the .NET Core SDK" } diff --git a/eng/docker-tools/Invoke-CleanupDocker.ps1 b/eng/docker-tools/Invoke-CleanupDocker.ps1 new file mode 100644 index 000000000..ad637872c --- /dev/null +++ b/eng/docker-tools/Invoke-CleanupDocker.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +docker ps -a -q | ForEach-Object { docker rm -f $_ } + +docker volume prune -f + +# Preserve the tagged Windows base images and the common eng infra images (e.g. ImageBuilder) +# to avoid the expense of having to repull continuously. +$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1 + +docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | + Where-Object { + $localImage = $_ + $localImage.Contains(": ")` + -Or -Not ($localImage.StartsWith("mcr.microsoft.com/windows")` + -Or ($imageNameVars.Values.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } | + ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } | + Select-Object -Unique | + ForEach-Object { docker rmi -f $_ } diff --git a/eng/docker-tools/Invoke-ImageBuilder.ps1 b/eng/docker-tools/Invoke-ImageBuilder.ps1 new file mode 100644 index 000000000..4cec5ba2b --- /dev/null +++ b/eng/docker-tools/Invoke-ImageBuilder.ps1 @@ -0,0 +1,105 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Executes ImageBuilder with the specified args. + +.PARAMETER ImageBuilderArgs +The args to pass to ImageBuilder. + +.PARAMETER ReuseImageBuilderImage +Indicates that a previously built ImageBuilder image is presumed to exist locally and that +it should be used for this execution of the script. This allows some optimization when +multiple calls are being made to this script that don't require a fresh image (i.e. the +repo contents in the image don't need to be or should not be updated with each call to +this script). + +.PARAMETER OnCommandExecuted +A ScriptBlock that will be invoked after the ImageBuilder command has been executed. +This allows the caller to execute extra logic in the context of the ImageBuilder while +its container is still running. +The ScriptBlock is passed the following argument values: + 1. Container name +#> +[cmdletbinding()] +param( + [string] + $ImageBuilderArgs, + + [switch] + $ReuseImageBuilderImage, + + [scriptblock] + $OnCommandExecuted +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + $host.SetShouldExit($LASTEXITCODE) + exit $LASTEXITCODE + throw "Failed: '$Cmd'" + } +} + +$imageBuilderContainerName = "ImageBuilder-$(Get-Date -Format yyyyMMddhhmmss)" +$containerCreated = $false + +pushd $PSScriptRoot/../../ +try { + $activeOS = docker version -f "{{ .Server.Os }}" + if ($activeOS -eq "linux") { + # On Linux, ImageBuilder is run within a container. + $imageBuilderImageName = "microsoft-dotnet-imagebuilder-withrepo" + if ($ReuseImageBuilderImage -ne $True) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec ("docker build -t $imageBuilderImageName --build-arg " ` + + "IMAGE=${imageNames.imageBuilderName} -f eng/docker-tools/Dockerfile.WithRepo .") + } + + $imageBuilderCmd = "docker run --name $imageBuilderContainerName -v /var/run/docker.sock:/var/run/docker.sock $imageBuilderImageName" + $containerCreated = $true + } + else { + # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container. + # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved + $imageBuilderFolder = ".Microsoft.DotNet.ImageBuilder" + $imageBuilderCmd = [System.IO.Path]::Combine($imageBuilderFolder, "Microsoft.DotNet.ImageBuilder.exe") + if (-not (Test-Path -Path "$imageBuilderCmd" -PathType Leaf)) { + & ./eng/docker-tools/Get-ImageBuilder.ps1 + Exec "docker create --name $imageBuilderContainerName ${imageNames.imageBuilderName}" + $containerCreated = $true + if (Test-Path -Path $imageBuilderFolder) + { + Remove-Item -Recurse -Force -Path $imageBuilderFolder + } + + Exec "docker cp ${imageBuilderContainerName}:/image-builder $imageBuilderFolder" + } + } + + Exec "$imageBuilderCmd $ImageBuilderArgs" + + if ($OnCommandExecuted) { + Invoke-Command $OnCommandExecuted -ArgumentList $imageBuilderContainerName + } +} +finally { + if ($containerCreated) { + Exec "docker container rm -f $imageBuilderContainerName" + } + + popd +} diff --git a/eng/docker-tools/Invoke-WithRetry.ps1 b/eng/docker-tools/Invoke-WithRetry.ps1 new file mode 100644 index 000000000..0aa2c2737 --- /dev/null +++ b/eng/docker-tools/Invoke-WithRetry.ps1 @@ -0,0 +1,41 @@ +#!/usr/bin/env pwsh + +# Executes a command and retries if it fails. +[cmdletbinding()] +param ( + [Parameter(Mandatory = $true)][string]$Cmd, + [int]$Retries = 2, + [int]$WaitFactor = 6 + ) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$count = 0 +$completed = $false + +Write-Output "Executing '$Cmd'" + +while (-not $completed) { + try { + Invoke-Expression $Cmd + if (-not $(Test-Path variable:LASTEXITCODE) -or $LASTEXITCODE -eq 0) { + $completed = $true + continue + } + } + catch { + } + + $count++ + + if ($count -lt $Retries) { + $wait = [Math]::Pow($WaitFactor, $count - 1) + Write-Output "Retry $count/$Retries, retrying in $wait seconds..." + Start-Sleep $wait + } + else { + Write-Output "Retry $count/$Retries, no more retries left." + throw "Failed to execute '$Cmd'" + } +} diff --git a/eng/docker-tools/Pull-Image.ps1 b/eng/docker-tools/Pull-Image.ps1 new file mode 100644 index 000000000..2d0a82cf5 --- /dev/null +++ b/eng/docker-tools/Pull-Image.ps1 @@ -0,0 +1,18 @@ +#!/usr/bin/env pwsh + +[cmdletbinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Image, + + [Parameter(Mandatory = $false)] + [int]$Retries = 2, + + [Parameter(Mandatory = $false)] + [int]$WaitFactor = 6 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/eng/docker-tools/Retain-Build.ps1 b/eng/docker-tools/Retain-Build.ps1 new file mode 100644 index 000000000..7f836e121 --- /dev/null +++ b/eng/docker-tools/Retain-Build.ps1 @@ -0,0 +1,43 @@ +# Adapted from https://github.com/dotnet/arcade/blob/main/eng/docker-tools/retain-build.ps1 +Param( + [Parameter(Mandatory = $true)][int] $BuildId, + [Parameter(Mandatory = $true)][string] $AzdoOrgUri, + [Parameter(Mandatory = $true)][string] $AzdoProject, + [Parameter(Mandatory = $true)][string] $Token +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +function Get-AzDOHeaders( + [string] $Token) { + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${Token}")) + $headers = @{"Authorization" = "Basic $base64AuthInfo" } + return $headers +} + +function Update-BuildRetention( + [string] $AzdoOrgUri, + [string] $AzdoProject, + [int] $BuildId, + [string] $Token) { + $headers = Get-AzDOHeaders -Token $Token + $requestBody = "{ + `"keepForever`": `"true`" + }" + + $requestUri = "${AzdoOrgUri}/${AzdoProject}/_apis/build/builds/${BuildId}?api-version=6.0" + Write-Host "Attempting to retain build using the following URI: ${requestUri} ..." + + try { + Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" + Write-Host "Updated retention settings for build ${BuildId}." + } + catch { + Write-Host "##[error] Failed to update retention settings for build: $($_.Exception.Response.StatusDescription)" + exit 1 + } +} + +Update-BuildRetention -AzdoOrgUri $AzdoOrgUri -AzdoProject $AzdoProject -BuildId $BuildId -Token $Token +exit 0 diff --git a/eng/docker-tools/build.ps1 b/eng/docker-tools/build.ps1 new file mode 100644 index 000000000..13c0806fb --- /dev/null +++ b/eng/docker-tools/build.ps1 @@ -0,0 +1,76 @@ +#!/usr/bin/env pwsh + +<# + .SYNOPSIS + Builds the Dockerfiles +#> + +[cmdletbinding()] +param( + # Product versions to filter by + [string[]]$Version = "*", + + # Names of OS to filter by + [string[]]$OS, + + # Type of architecture to filter by + [string]$Architecture, + + # Additional custom path filters + [string[]]$Paths, + + # Path to manifest file + [string]$Manifest = "manifest.json", + + # Additional args to pass to ImageBuilder + [string]$OptionalImageBuilderArgs +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Log { + param ([string] $Message) + + Write-Output $Message +} + +function Exec { + param ([string] $Cmd) + + Log "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + throw "Failed: '$Cmd'" + } +} + +pushd $PSScriptRoot/../.. +try { + $args = $OptionalImageBuilderArgs + + if ($Version) { + $args += ($Version | foreach { ' --version "{0}"' -f $_ }) + } + + if ($OS) { + $args += ($OS | foreach { ' --os-version "{0}"' -f $_ }) + } + + if ($Architecture) { + $args += ' --architecture "{0}"' -f $Architecture + } + + if ($Paths) { + $args += ($Paths | foreach { ' --path "{0}"' -f $_ }) + } + + if ($Manifest) { + $args += ' --manifest "{0}"' -f $Manifest + } + + ./eng/docker-tools/Invoke-ImageBuilder.ps1 "build $args" +} +finally { + popd +} diff --git a/eng/docker-tools/readme.md b/eng/docker-tools/readme.md new file mode 100644 index 000000000..c0fe1f2eb --- /dev/null +++ b/eng/docker-tools/readme.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools. \ No newline at end of file diff --git a/eng/docker-tools/templates/1es-official.yml b/eng/docker-tools/templates/1es-official.yml new file mode 100644 index 000000000..ebf8fcd70 --- /dev/null +++ b/eng/docker-tools/templates/1es-official.yml @@ -0,0 +1,70 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. +# +# The `cgDryRun` parameter will run CG but not submit the results, for testing purposes. + +parameters: +- name: cgDryRun + type: boolean + default: false +- name: stages + type: stageList + default: [] +- name: serviceConnections + type: object + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: ${{ parameters.cgDryRun }} + showAlertLink: true + policheck: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: VersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ if gt(length(parameters.serviceConnections), 0) }}: + - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es-unofficial.yml b/eng/docker-tools/templates/1es-unofficial.yml new file mode 100644 index 000000000..bc584cd38 --- /dev/null +++ b/eng/docker-tools/templates/1es-unofficial.yml @@ -0,0 +1,79 @@ +# This unofficial template will always run CG in "what if" mode, which will not submit results to the CG. SDL tools may +# also be disabled for testing purposes. +# +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: disableSDL + type: boolean + default: false + displayName: Disable SDL +- name: stages + type: stageList + default: [] + # 1ES Pipeline Template parameters +- name: serviceConnections + type: object + default: [] +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + # Use a unique task prefix for unofficial pipelines + taskPrefix: "🟦" + baseTemplate: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + whatIf: true + showAlertLink: true + enableAllTools: ${{ not(parameters.disableSDL) }} + policheck: + enabled: true + sbom: + enabled: true + sourceRepositoriesToScan: + exclude: + - repository: InternalVersionsRepo + - repository: PublicVersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ if gt(length(parameters.serviceConnections), 0) }}: + - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es.yml b/eng/docker-tools/templates/1es.yml new file mode 100644 index 000000000..5fde4a73d --- /dev/null +++ b/eng/docker-tools/templates/1es.yml @@ -0,0 +1,75 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: stages + type: stageList + default: [] +# List of repositories that will be excluded from SDL scanning. This should +# only be used when including other repos without building their source code. +# E.g. for the dotnet/versions repo. +- name: reposToExcludeFromScanning + type: object + default: [] +# The pool that will be used for initializing service connections. +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# The pool that will be used for SDL jobs. +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows +# Container image SBOMs are generated manually during the build job. 1ESPT's +# automatic SBOM generation only adds unnecessary steps and artifacts to +# builds. SBOM is not needed for JSON outputs. If a pipeline outputs binary +# artifacts that ship to customers, then set this parameter to true. +- name: enableSbom + type: boolean + default: false + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/docker-tools/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + sbom: + enabled: ${{ parameters.enableSbom }} + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + policheck: + enabled: true + ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: + sourceRepositoriesToScan: + exclude: + - ${{ each repo in parameters.reposToExcludeFromScanning }}: + - repository: ${{ repo }} + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml new file mode 100644 index 000000000..1f7f9b566 --- /dev/null +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -0,0 +1,131 @@ +parameters: + name: null + pool: {} + matrix: {} + dockerClientOS: null + buildJobTimeout: 60 + commonInitStepsForMatrixAndBuild: [] + customInitSteps: [] + publishConfig: null + noCache: false + internalProjectName: null + publicProjectName: null + storageAccountServiceConnection: null + +jobs: +- job: ${{ parameters.name }} + condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + dependsOn: + - PreBuildValidation + - CopyBaseImages + - GenerateBuildMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.buildJobTimeout }} + variables: + imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions) + sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom + imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo + imageInfoContainerDir: $(artifactsPath)/imageInfo + steps: + - ${{ parameters.commonInitStepsForMatrixAndBuild }} + - template: /eng/docker-tools/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self + parameters: + cleanupDocker: true + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + - powershell: echo "##vso[task.setvariable variable=imageBuilderBuildArgs]" + condition: eq(variables.imageBuilderBuildArgs, '') + displayName: Initialize Image Builder Build Args + - powershell: | + New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force + + # Reference the existing imageBuilderBuildArgs variable as an environment variable rather than injecting it directly + # with the $(imageBuilderBuildArgs) syntax. This is to avoid issues where the string may contain single quotes $ chars + # which really mess up assigning to a variable. It would require assigning the string with single quotes but also needing + # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since + # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates + # the environment variable for us. + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push" + } + + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" + } + + echo "imageBuilderBuildArgs: $imageBuilderBuildArgs" + echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs" + displayName: Set Image Builder Build Args + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: BuildImages + displayName: Build Images + serviceConnections: + # "name" here refers to the argument name, not the service connection name. + # It should probably be changed to "argName". + - name: acr + id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }} + - ${{ if parameters.storageAccountServiceConnection }}: + - name: storage + id: ${{ parameters.storageAccountServiceConnection.id }} + tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }} + clientId: ${{ parameters.storageAccountServiceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + dockerClientOS: ${{ parameters.dockerClientOS }} + args: >- + build + --manifest $(manifest) + $(imageBuilderPaths) + $(osVersions) + --os-type $(osType) + --architecture $(architecture) + --retry + --digests-out-var 'builtImages' + --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}' + --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + $(manifestVariables) + $(imageBuilderBuildArgs) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: $(legName)-image-info-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - powershell: | + $images = "$(BuildImages.builtImages)" + if (-not $images) { return 0 } + $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" + & $(engDockerToolsPath)/Pull-Image.ps1 $syftImageName + $images -Split ',' | ForEach-Object { + echo "Generating SBOM for $_"; + $targetImageName = "$_"; + $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); + $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; + New-Item -Type Directory -Path $sbomChildDir > $null; + docker build --output=$sbomChildDir -f $(engDockerToolsPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engDockerToolsPath); + } + displayName: Generate SBOMs + condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/docker-tools/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self + parameters: + condition: ne(variables.testScriptPath, '') + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sbomDirectory) + artifactName: $(legName)-sboms + displayName: Publish SBOM + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + condition: ne(variables['BuildImages.builtImages'], '') diff --git a/eng/docker-tools/templates/jobs/cg-build-projects.yml b/eng/docker-tools/templates/jobs/cg-build-projects.yml new file mode 100644 index 000000000..845a6719c --- /dev/null +++ b/eng/docker-tools/templates/jobs/cg-build-projects.yml @@ -0,0 +1,58 @@ +# This job builds all projects in the repository. It is intended to be used for CG purposes. +# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they +# need to be built outside of Dockerfiles. +parameters: +# Setting cgDryRun will run CG but not submit the results +- name: cgDryRun + type: boolean + default: false + displayName: CG Dry Run +# See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values +- name: dotnetVersionChannel + type: string + default: '9.0' + displayName: .NET Version + +jobs: +- job: BuildProjects + displayName: Build Projects + pool: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux + steps: + - powershell: > + ./eng/docker-tools/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "/usr/share/.dotnet" + displayName: Run Dotnet Install Script + - script: > + find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build + displayName: Build Projects + + # Component Detection is only automatically run on production branches. + # To run Component Detection on non-production branches, the task must be manually injected. + - ${{ if eq(parameters.cgDryRun, true) }}: + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)" + Write-Host "##vso[build.addbuildtag]dry-run" + + if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)")) + { + Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))." + Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results." + exit 1 + } + displayName: Update Build Number + - task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (manually injected) + inputs: + # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show + # which components would trigger an alert on a production build. + # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components + # to CG for analysis. Only the results for tracked branches matter. + scanType: Register + whatIf: false + alertWarningLevel: Low + failOnAlert: false + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + timeoutInMinutes: 10 diff --git a/eng/docker-tools/templates/jobs/copy-base-images-staging.yml b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml new file mode 100644 index 000000000..20ce1f4da --- /dev/null +++ b/eng/docker-tools/templates/jobs/copy-base-images-staging.yml @@ -0,0 +1,29 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: publishConfig + type: object + default: null +- name: customInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false + +jobs: +- template: /eng/docker-tools/templates/jobs/copy-base-images.yml@self + parameters: + name: ${{ parameters.name }} + pool: ${{ parameters.pool }} + customInitSteps: ${{ parameters.customInitSteps }} + additionalOptions: ${{ parameters.additionalOptions }} + acr: ${{ parameters.publishConfig.internalMirrorAcr }} + repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} diff --git a/eng/docker-tools/templates/jobs/copy-base-images.yml b/eng/docker-tools/templates/jobs/copy-base-images.yml new file mode 100644 index 000000000..ffc5aee15 --- /dev/null +++ b/eng/docker-tools/templates/jobs/copy-base-images.yml @@ -0,0 +1,39 @@ +parameters: +- name: name + type: string + default: null +- name: pool + type: object + default: {} +- name: acr + type: object + default: null +- name: repoPrefix + type: string + default: null +- name: customInitSteps + type: stepList + default: [] +- name: additionalOptions + type: string + default: '' +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self + parameters: + acr: ${{ parameters.acr }} + repoPrefix: ${{ parameters.repoPrefix }} + additionalOptions: ${{ parameters.additionalOptions }} + continueOnError: ${{ parameters.continueOnError }} + forceDryRun: ${{ parameters.forceDryRun }} diff --git a/eng/docker-tools/templates/jobs/generate-matrix.yml b/eng/docker-tools/templates/jobs/generate-matrix.yml new file mode 100644 index 000000000..606dec57f --- /dev/null +++ b/eng/docker-tools/templates/jobs/generate-matrix.yml @@ -0,0 +1,72 @@ +parameters: + matrixType: null + name: null + pool: {} + customBuildLegGroupArgs: "" + isTestStage: false + internalProjectName: null + noCache: false + publishConfig: null + customInitSteps: [] + commonInitStepsForMatrixAndBuild: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + pool: ${{ parameters.pool }} + steps: + - ${{ parameters.commonInitStepsForMatrixAndBuild }} + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml + parameters: + publicSourceBranch: $(publicSourceBranch) + - ${{ if eq(parameters.isTestStage, true) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - powershell: | + $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)" + + if ("${{ parameters.isTestStage}}" -eq "true") { + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" + } + elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache + $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" + } + + echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions" + displayName: Set GenerateBuildMatrix Variables + - script: > + echo "##vso[task.setvariable variable=generateBuildMatrixCommand] + generateBuildMatrix + --manifest $(manifest) + --type ${{ parameters.matrixType }} + --os-type '*' + --architecture '*' + --product-version-components $(productVersionComponents) + ${{ parameters.customBuildLegGroupArgs }} + $(imageBuilder.pathArgs) + $(manifestVariables) + $(commonMatrixAndBuildOptions) + $(additionalGenerateBuildMatrixOptions)" + displayName: Set GenerateBuildMatrix Command + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + name: matrix + displayName: Generate ${{ parameters.matrixType }} Matrix + serviceConnections: + - name: acr + tenantId: $(build.serviceConnection.tenantId) + clientId: $(build.serviceConnection.clientId) + id: $(build.serviceConnection.id) + internalProjectName: internal + args: $(generateBuildMatrixCommand) diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml new file mode 100644 index 000000000..33b91ec03 --- /dev/null +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -0,0 +1,87 @@ +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + +jobs: +- job: Build + pool: ${{ parameters.pool }} + variables: + imageInfosSubDir: "/image-infos" + imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" + imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" + imageInfosOutputSubDir: "/output" + sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" + steps: + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + # This can fail if no build jobs ran to produce any artifacts + continueOnError: true + - powershell: | + # Move all image-info artifacts to their own directory + New-Item -ItemType Directory -Path $(imageInfosHostDir) + Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) | + Move-Item -Verbose -Destination $(imageInfosHostDir) + displayName: Collect Image Info Files + - powershell: | + # Move the contents of all the SBOM artifact directories to a single location + New-Item -ItemType Directory -Path $(sbomOutputDir) + Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) | + ForEach-Object { + Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(sbomOutputDir) + } + displayName: Consolidate SBOMs to Single Directory + - powershell: | + # Deletes the artifacts from all the unsuccessful jobs + Get-ChildItem $(imageInfosHostDir) -Directory | + ForEach-Object { + [pscustomobject]@{ + # Parse the artifact name to separate the base of the name from the job attempt number + BaseName = $_.Name.Substring(0, $_.Name.LastIndexOf('-')); + JobAttempt = $_.Name.Substring($_.Name.LastIndexOf('-') + 1) + FullName = $_.FullName + } + } | + Group-Object BaseName | + # Delete all but the last artifact from each base name + ForEach-Object { + $_.Group | + Sort-Object JobAttempt | + Select-Object -ExpandProperty FullName -SkipLast 1 | + Remove-Item -Recurse -Force + } + displayName: Prune Publish Artifacts + - powershell: | + $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)" + if ($imageInfoFiles.Count -eq 0) { + echo "No image info files found." + echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true" + exit 0 + } + + New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force + $(runImageBuilderCmd) mergeImageInfo ` + --manifest $(manifest) ` + $(imageInfosContainerDir) ` + $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` + $(manifestVariables) + name: MergeImageInfoFiles + displayName: Merge Image Info Files + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(sbomOutputDir) + artifactName: sboms + displayName: Publish SBOM Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true')) + path: $(imageInfosHostDir)$(imageInfosOutputSubDir) + artifactName: image-info + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml new file mode 100644 index 000000000..cb9b454ac --- /dev/null +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -0,0 +1,312 @@ +parameters: + pool: {} + internalProjectName: null + publishConfig: null + customInitSteps: [] + customPublishVariables: [] + sourceBuildPipelineDefinitionId: "" + sourceBuildPipelineRunId: "" + versionsRepoRef: null + versionsRepoPath: "" + # When true, overrides the commit SHA in merged image info files to use the current repository commit. + # This ensures that updated images reference the correct commit in their commitUrl properties. + overrideImageInfoCommit: false + +jobs: +- job: Publish + pool: ${{ parameters.pool }} + timeoutInMinutes: 90 + + variables: + - name: imageBuilder.commonCmdArgs + value: >- + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' + $(manifestVariables) + $(imageBuilder.queueArgs) + - name: publishNotificationRepoName + value: $(Build.Repository.Name) + - name: branchName + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}: + value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] + - name: imageInfoHostDir + value: $(Build.ArtifactStagingDirectory)/imageInfo + - name: imageInfoContainerDir + value: $(artifactsPath)/imageInfo + - name: sourceBuildIdOutputDir + value: $(Build.ArtifactStagingDirectory)/sourceBuildId + - name: commitOverrideArg + ${{ if eq(parameters.overrideImageInfoCommit, true) }}: + value: --commit-override $(Build.SourceVersion) + ${{ else }}: + value: '' + - ${{ parameters.customPublishVariables }} + + steps: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/steps/retain-build.yml@self + + - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + + - pwsh: | + $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI + echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName" + $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}" + echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot" + displayName: Set Publish Variables + + - ${{ parameters.customInitSteps }} + + - template: /eng/docker-tools/templates/steps/validate-branch.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoHostDir) + artifactName: image-info + piplineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + # This can fail in scenarios where no build jobs have run to produce any artifacts + continueOnError: true + + - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self + parameters: + publicSourceBranch: $(publicSourceBranch) + + - template: /eng/docker-tools/templates/steps/set-dry-run.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + + - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" + displayName: Set Publish Variables + + - script: > + $(runImageBuilderCmd) trimUnchangedPlatforms + '$(imageInfoContainerDir)/image-info.json' + displayName: Trim Unchanged Images + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Images + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + copyAcrImages + '${{ parameters.publishConfig.buildAcr.subscription }}' + '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + '${{ parameters.publishConfig.buildAcr.repoPrefix }}' + '${{ parameters.publishConfig.buildAcr.server }}' + --os-type '*' + --architecture '*' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --image-info '$(imageInfoContainerDir)/image-info.json' + $(dryRunArg) + $(imageBuilder.pathArgs) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Publish Manifest + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + dockerClientOS: ${{ parameters.dockerClientOS }} + args: >- + publishManifest + '$(imageInfoContainerDir)/image-info.json' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --os-type '*' + --architecture '*' + $(dryRunArg) + $(imageBuilder.pathArgs) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(imageInfoHostDir) + artifactName: image-info-final-$(System.JobAttempt) + displayName: Publish Image Info File Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + + - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + imageInfoPath: '$(imageinfoContainerDir)/image-info.json' + minQueueTime: $(imageQueueTime) + dryRunArg: $(dryRunArg) + condition: succeeded() + + - template: /eng/docker-tools/templates/steps/publish-readmes.yml@self + parameters: + dryRunArg: $(dryRunArg) + condition: and(succeeded(), eq(variables['publishReadme'], 'true')) + + - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data + displayName: Create EOL Annotation Data Directory + + - script: |- + cd $(versionsRepoRoot) + git pull origin $(gitHubVersionsRepoInfo.branch) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Pull Latest Changes from Versions Repo + + - script: >- + cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Copy Latest Image Info from Versions Repo + + - script: > + $(runImageBuilderCmd) mergeImageInfo + $(imageInfoContainerDir) + $(imageInfoContainerDir)/full-image-info-new.json + $(manifestVariables) + $(dryRunArg) + --manifest $(manifest) + --publish + --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json + $(commitOverrideArg) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Merge Image Info + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Ingest Kusto Image Info + serviceConnections: + - name: kusto + id: $(kusto.serviceConnection.id) + tenantId: $(kusto.serviceConnection.tenantId) + clientId: $(kusto.serviceConnection.clientId) + internalProjectName: ${{ parameters.internalProjectName }} + condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) + args: >- + ingestKustoImageInfo + '$(imageInfoContainerDir)/image-info.json' + '$(kusto.cluster)' + '$(kusto.database)' + '$(kusto.imageTable)' + '$(kusto.layerTable)' + --os-type '*' + --architecture '*' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Generate EOL Annotation Data + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + generateEolAnnotationDataForPublish + '${{ parameters.publishConfig.publishAcr.server }}' + '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' + '$(imageInfoContainerDir)/full-image-info-orig.json' + '$(imageInfoContainerDir)/full-image-info-new.json' + $(generateEolAnnotationDataExtraOptions) + $(dryRunArg) + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/eol-annotation-data + artifactName: eol-annotation-data-$(System.JobAttempt) + displayName: Publish EOL Annotation Data Artifact + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + + - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self + parameters: + acr: ${{ parameters.publishConfig.publishAcr }} + dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + $(gitHubVersionsRepoInfo.authArgs) + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info + + # Task displayNames names are hardcoded to reference the task prefix used by 1ES official + # pipelines in eng/docker-tools/templates/1es-official.yml. + # + # These will fail if they are dependend on by an unofficial pipeline since they use a unique task + # prefix compared to official pipelines (see eng/docker-tools/templates/1es-unofficial.yml). This is + # acceptable because unofficial pipelines should not publish images. + # + # https://github.com/dotnet/docker-tools/issues/1698 tracks making this command no longer depend + # on individual step displayNames. + - script: > + $(runImageBuilderCmd) postPublishNotification + '$(publishNotificationRepoName)' + '$(branchName)' + '$(imageInfoContainerDir)/image-info.json' + $(Build.BuildId) + '$(System.AccessToken)' + '$(azdoOrgName)' + '$(System.TeamProject)' + $(gitHubNotificationsRepoInfo.authArgs) + '$(gitHubNotificationsRepoInfo.org)' + '$(gitHubNotificationsRepoInfo.repo)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --task "🟪 Copy Images" + --task "🟪 Publish Manifest" + --task "🟪 Wait for Image Ingestion" + --task "🟪 Publish Readmes" + --task "🟪 Wait for MCR Doc Ingestion" + --task "🟪 Publish Image Info" + --task "🟪 Ingest Kusto Image Info" + --task "🟪 Generate EOL Annotation Data" + --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.publishAcr.server }})" + --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.publishAcr.server }})" + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + displayName: Post Publish Notification + condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true')) + + - powershell: | + # Default to current build number if parameter was not overridden + $buildId = "${{ parameters.sourceBuildPipelineRunId }}" + if ($buildId -eq "") { + $buildId = "$(Build.BuildNumber)" + } + + New-Item -ItemType Directory -Path $(sourceBuildIdOutputDir) + Set-Content -Path $(sourceBuildIdOutputDir)/source-build-id.txt -Value "$buildId" + condition: succeeded() + displayName: Write Source Build ID to File + + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(sourceBuildIdOutputDir) + artifactName: source-build-id + displayName: Publish Source Build ID Artifact + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} diff --git a/eng/docker-tools/templates/jobs/test-images-linux-client.yml b/eng/docker-tools/templates/jobs/test-images-linux-client.yml new file mode 100644 index 000000000..44e9e2fb7 --- /dev/null +++ b/eng/docker-tools/templates/jobs/test-images-linux-client.yml @@ -0,0 +1,30 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + preBuildValidation: false + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + ${{ if eq(parameters.preBuildValidation, 'false') }}: + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + strategy: + matrix: $[ ${{ parameters.matrix }} ] + ${{ if eq(parameters.preBuildValidation, 'true') }}: + condition: and(succeeded(), ne(variables.testScriptPath, '')) + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-linux-client.yml@self + parameters: + preBuildValidation: ${{ parameters.preBuildValidation }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/jobs/test-images-windows-client.yml b/eng/docker-tools/templates/jobs/test-images-windows-client.yml new file mode 100644 index 000000000..390f01da6 --- /dev/null +++ b/eng/docker-tools/templates/jobs/test-images-windows-client.yml @@ -0,0 +1,25 @@ +parameters: + name: null + pool: {} + matrix: {} + testJobTimeout: 60 + internalProjectName: null + publishConfig: null + customInitSteps: [] + sourceBuildPipelineRunId: "" + +jobs: +- job: ${{ parameters.name }} + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + pool: ${{ parameters.pool }} + strategy: + matrix: $[ ${{ parameters.matrix }} ] + timeoutInMinutes: ${{ parameters.testJobTimeout }} + steps: + - template: /eng/docker-tools/templates/steps/test-images-windows-client.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml new file mode 100644 index 000000000..b9df61a4c --- /dev/null +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -0,0 +1,343 @@ +parameters: + buildMatrixType: platformDependencyGraph + testMatrixType: platformVersionedOs + buildMatrixCustomBuildLegGroupArgs: "" + testMatrixCustomBuildLegGroupArgs: "" + customCopyBaseImagesInitSteps: [] + customGenerateMatrixInitSteps: [] + customBuildInitSteps: [] + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + + noCache: false + publishConfig: null + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: "" + + storageAccountServiceConnection: null + + linuxAmd64Pool: + vmImage: $(defaultLinuxAmd64PoolImage) + linuxArm32Pool: + vmImage: $(defaultLinuxArm32PoolImage) + linuxArm64Pool: + vmImage: $(defaultLinuxArm64PoolImage) + windows2016Pool: + vmImage: $(defaultWindows2016PoolImage) + windows1809Pool: + vmImage: $(defaultWindows1809PoolImage) + windows2022Pool: + vmImage: $(defaultWindows2022PoolImage) + windows2025Pool: + vmImage: $(defaultWindows2025PoolImage) + + +################################################################################ +# Build Images +################################################################################ +stages: +- stage: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + dependsOn: [] + jobs: + + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables + + - template: /eng/docker-tools/templates/jobs/copy-base-images-staging.yml@self + parameters: + name: CopyBaseImages + publishConfig: ${{ parameters.publishConfig }} + pool: ${{ parameters.linuxAmd64Pool }} + additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" + customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.buildMatrixType }} + name: GenerateBuildMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + internalProjectName: ${{ parameters.internalProjectName }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxAmd64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm64'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.LinuxArm32'] + dockerClientOS: linux + buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.Windows1809Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + - template: /eng/docker-tools/templates/jobs/build-images.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateBuildMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + dockerClientOS: windows + buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + customInitSteps: ${{ parameters.customBuildInitSteps }} + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + +################################################################################ +# Post-Build +################################################################################ +- stage: Post_Build + dependsOn: Build + condition: and(succeeded(), contains(variables['stages'], 'build')) + jobs: + - template: /eng/docker-tools/templates/jobs/post-build.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + +################################################################################ +# Test Images +################################################################################ +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Test + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + ne(variables['testScriptPath'], ''), + and( + contains(variables['stages'], 'test'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/generate-matrix.yml@self + parameters: + matrixType: ${{ parameters.testMatrixType }} + name: GenerateTestMatrix + pool: ${{ parameters.linuxAmd64Pool }} + customBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + isTestStage: true + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + commonInitStepsForMatrixAndBuild: + - template: /eng/docker-tools/templates/steps/init-matrix-build-publish.yml@self + parameters: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + publishConfig: ${{ parameters.publishConfig }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_amd64 + pool: ${{ parameters.linuxAmd64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm64 + pool: ${{ parameters.linuxArm64Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-linux-client.yml@self + parameters: + name: Linux_arm32 + pool: ${{ parameters.linuxArm32Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] + testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows1809_amd64 + pool: ${{ parameters.windows1809Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2022_amd64 + pool: ${{ parameters.windows2022Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: Windows2025_amd64 + pool: ${{ parameters.windows2025Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + - template: /eng/docker-tools/templates/jobs/test-images-windows-client.yml@self + parameters: + name: WindowsLtsc2016_amd64 + pool: ${{ parameters.windows2016Pool }} + matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] + testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customInitSteps: ${{ parameters.customTestInitSteps }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml new file mode 100644 index 000000000..311b16133 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml @@ -0,0 +1,133 @@ +# A wrapper template around the common build-test-publish-repo template with settings +# specific to the .NET team's infrastructure. + +parameters: + linuxAmd64Pool: "" + + # (Optional) This service connection should be an Azure Resource Manager + # service connection to a storage account that's needed during image builds. + # It can be used to build images with access to private/internal bits. + # If specified, this service connection will be used to pass a storage + # account access token as `--build-arg ACCESSTOKEN=***` to all image builds. + storageAccountServiceConnection: null + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + internalProjectName: null + publicProjectName: null + + versionsRepoRef: null + +stages: +- template: /eng/docker-tools/templates/stages/build-and-test.yml@self + parameters: + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + buildMatrixType: ${{ parameters.buildMatrixType }} + testMatrixType: ${{ parameters.testMatrixType }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + # Only clone versions repo if we need to reference it during the build in order to cache images. + ${{ if eq(parameters.noCache, false) }}: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + + # Linux AMD64 + linuxAmd64Pool: + ${{ if ne(parameters.linuxAmd64Pool, '') }}: + ${{ parameters.linuxAmd64Pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + vmImage: $(defaultLinuxAmd64PoolImage) + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux + + # Linux Arm64 + linuxArm64Pool: + os: linux + hostArchitecture: Arm64 + image: $(linuxArm64PoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm64PublicPoolName) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm64InternalPoolName) + + # Linux Arm32 + linuxArm32Pool: + os: linux + hostArchitecture: Arm64 + image: $(linuxArm32PoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + name: $(linuxArm32PublicPoolName) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxArm32InternalPoolName) + + # Windows Server 2016 + windows2016Pool: + os: windows + name: $(windowsServer2016PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2016PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2016InternalPoolImage) + + # Windows Server 2019 (1809) + windows1809Pool: + os: windows + name: $(windowsServer2019PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2019PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2019InternalPoolImage) + + # Windows Server 2022 + windows2022Pool: + os: windows + name: $(windowsServer2022PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2022PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2022InternalPoolImage) + + # Windows Server 2025 + windows2025Pool: + os: windows + name: $(windowsServer2025PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2025PublicPoolImage) + ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + image: $(windowsServer2025InternalPoolImage) diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml new file mode 100644 index 000000000..c412f00d2 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -0,0 +1,74 @@ +# This template wraps the .NET-specific build-and-test and publish templates + +parameters: + linuxAmd64Pool: "" + + # Parameters for pre-build jobs + customGenerateMatrixInitSteps: [] + customCopyBaseImagesInitSteps: [] + + # Build parameters + noCache: false + publishConfig: null + buildMatrixType: platformDependencyGraph + buildMatrixCustomBuildLegGroupArgs: "" + linuxAmdBuildJobTimeout: 60 + linuxArmBuildJobTimeout: 60 + windowsAmdBuildJobTimeout: 60 + customBuildInitSteps: [] + + # Test parameters + testMatrixType: platformVersionedOs + testMatrixCustomBuildLegGroupArgs: "" + linuxAmdTestJobTimeout: 60 + linuxArmTestJobTimeout: 60 + windowsAmdTestJobTimeout: 60 + customTestInitSteps: [] + sourceBuildPipelineRunId: "" + + # Publish parameters + customPublishInitSteps: [] + + # Other common parameters + internalProjectName: null + publicProjectName: null + versionsRepoRef: "" + +stages: +- template: /eng/docker-tools/templates/stages/dotnet/build-and-test.yml@self + parameters: + linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} + # Pre-build + customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} + customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + # Build + noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} + buildMatrixType: ${{ parameters.buildMatrixType }} + buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} + linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} + linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + # Test + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + testMatrixType: ${{ parameters.testMatrixType }} + testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} + linuxAmdTestJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + linuxArmTestJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + customTestInitSteps: ${{ parameters.customTestInitSteps }} + # Other + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + +- template: /eng/docker-tools/templates/stages/dotnet/publish.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + customPublishInitSteps: ${{ parameters.customPublishInitSteps }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml new file mode 100644 index 000000000..eddd78593 --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -0,0 +1,104 @@ +# This pipeline template injects the publish config for the dotnet-docker +# non-production (unofficial) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-prod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging-test.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(internal-mirror-test.serviceConnectionName) + id: $(internal-mirror-test.serviceConnection.id) + clientId: $(internal-mirror-test.serviceConnection.clientId) + tenantId: $(testTenant) + + publicMirrorAcr: + server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + + buildAcr: + server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + + cleanServiceConnection: + name: $(clean-test.serviceConnectionName) + id: $(clean-test.serviceConnection.id) + clientId: $(clean-test.serviceConnection.clientId) + tenantId: $(testTenant) + + testServiceConnection: + name: $(test-nonprod.serviceConnectionName) + id: $(test-nonprod.serviceConnection.id) + clientId: $(test-nonprod.serviceConnection.clientId) + tenantId: $(testTenant) + + publishAcr: + server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml new file mode 100644 index 000000000..d45807ddb --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -0,0 +1,104 @@ +# This pipeline template injects the publish config for the dotnet-docker +# production (official) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-nonprod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(internal-mirror.serviceConnectionName) + id: $(internal-mirror.serviceConnection.id) + clientId: $(internal-mirror.serviceConnection.clientId) + tenantId: $(internal-mirror.serviceConnection.tenantId) + + publicMirrorAcr: + server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + + buildAcr: + server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + + cleanServiceConnection: + name: $(clean.serviceConnectionName) + id: $(clean.serviceConnection.id) + clientId: $(clean.serviceConnection.clientId) + tenantId: $(clean.serviceConnection.tenantId) + + testServiceConnection: + name: $(test.serviceConnectionName) + id: $(test.serviceConnection.id) + clientId: $(test.serviceConnection.clientId) + tenantId: $(test.serviceConnection.tenantId) + + publishAcr: + server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/docker-tools/templates/stages/dotnet/publish.yml b/eng/docker-tools/templates/stages/dotnet/publish.yml new file mode 100644 index 000000000..66cb2c8bd --- /dev/null +++ b/eng/docker-tools/templates/stages/dotnet/publish.yml @@ -0,0 +1,56 @@ +# This template wraps the common publish stage template with settings specific +# to the .NET team's infrastructure. + +parameters: + internalProjectName: null + publicProjectName: null + publishConfig: null + pool: "" + isStandalonePublish: false + customPublishInitSteps: [] + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + versionsRepoRef: null + overrideImageInfoCommit: false + +stages: +- template: /eng/docker-tools/templates/stages/publish.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} + isStandalonePublish: ${{ parameters.isStandalonePublish }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + + customPublishInitSteps: + - pwsh: | + # When reporting the repo name in the publish notification, we don't want to include + # the org part of the repo name (e.g. we want "dotnet-docker", not "dotnet-dotnet-docker"). + # This also accounts for the different separators between AzDO and GitHub repo names. + + $repoName = "$(Build.Repository.Name)" + + $orgSeparatorIndex = $repoName.IndexOf("/") + if ($orgSeparatorIndex -eq -1) { + $orgSeparatorIndex = $repoName.IndexOf("-") + } + + if ($orgSeparatorIndex -ge 0) { + $repoName = $repoName.Substring($orgSeparatorIndex + 1) + } + echo "##vso[task.setvariable variable=publishNotificationRepoName]$repoName" + displayName: "Set Custom Repo Name Var" + - ${{ parameters.customPublishInitSteps }} + + pool: + ${{ if ne(parameters.pool, '') }}: + ${{ parameters.pool }} + ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + vmImage: $(defaultLinuxAmd64PoolImage) + ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) + os: linux diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml new file mode 100644 index 000000000..ffdd594c2 --- /dev/null +++ b/eng/docker-tools/templates/stages/publish.yml @@ -0,0 +1,80 @@ +parameters: + customPublishInitSteps: [] + customPublishVariables: [] + + internalProjectName: null + publicProjectName: null + + publishConfig: null + + isStandalonePublish: false + + pool: + vmImage: $(defaultLinuxAmd64PoolImage) + + sourceBuildPipelineDefinitionId: '' + sourceBuildPipelineRunId: '' + + versionsRepoRef: null + versionsRepoPath: "versions" + + # When true, any updated images will have the SHA in their commit URL updated + # to the commit that this pipeline is running on, instead of the commit they + # were built from. Use in combination with isStandalonePublish to ensure that + # internally built images still reference public Dockerfiles. + overrideImageInfoCommit: false + +################################################################################ +# Publish Images +################################################################################ +stages: +- stage: Publish + ${{ if eq(parameters.isStandalonePublish, true) }}: + dependsOn: [] + ${{ else }}: + ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Test + ${{ else }}: + dependsOn: Post_Build + condition: " + and( + not(canceled()), + and( + contains(variables['stages'], 'publish'), + or( + or( + and( + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + or( + and( + not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + and( + not(contains(variables['stages'], 'test')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build'))))), + not( + or( + contains(variables['stages'], 'build'), + contains(variables['stages'], 'test'))))))" + jobs: + - template: /eng/docker-tools/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customPublishInitSteps }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + versionsRepoPath: ${{ parameters.versionsRepoPath }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} diff --git a/eng/docker-tools/templates/stages/setup-service-connections.yml b/eng/docker-tools/templates/stages/setup-service-connections.yml new file mode 100644 index 000000000..2ef74e90c --- /dev/null +++ b/eng/docker-tools/templates/stages/setup-service-connections.yml @@ -0,0 +1,38 @@ +# This stage exists to tell Azure DevOps about all of the service connections +# that will be used in the pipeline. A service connection will not work unless +# it is declared in this stage's parameters, even if your pipeline has already +# been granted access to the service connection. This stage also does not need +# to complete before the service connection is used. +parameters: +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# serviceConnections object shape: +# - name: string +- name: serviceConnections + type: object + default: [] + +stages: + +- stage: SetupServiceConnectionsStage + displayName: Setup service connections + jobs: + + - job: SetupServiceConnectionsJob + displayName: Setup service connections + pool: ${{ parameters.pool }} + steps: + - checkout: none + - ${{ each serviceConnection in parameters.serviceConnections }}: + - task: AzureCLI@2 + displayName: Setup ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account show diff --git a/eng/docker-tools/templates/steps/annotate-eol-digests.yml b/eng/docker-tools/templates/steps/annotate-eol-digests.yml new file mode 100644 index 000000000..0cb090c1c --- /dev/null +++ b/eng/docker-tools/templates/steps/annotate-eol-digests.yml @@ -0,0 +1,48 @@ +parameters: +- name: acr + type: object +# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command +- name: dataFile + type: string + +steps: + - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests + displayName: Create Annotation Digests Directory + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Annotate EOL Images (${{ parameters.acr.server }}) + serviceConnections: + - name: acr + id: ${{ parameters.acr.serviceConnection.id }} + tenantId: ${{ parameters.acr.serviceConnection.tenantId }} + clientId: ${{ parameters.acr.serviceConnection.clientId }} + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: >- + annotateEolDigests + "${{ parameters.dataFile }}" + "${{ parameters.acr.server }}" + "${{ parameters.acr.repoPrefix }}" + $(artifactsPath)/annotation-digests/annotation-digests.txt + $(dryRunArg) + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/annotation-digests + artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) + displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) + args: >- + waitForMarAnnotationIngestion + $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/docker-tools/templates/steps/clean-acr-images.yml b/eng/docker-tools/templates/steps/clean-acr-images.yml new file mode 100644 index 000000000..b791de305 --- /dev/null +++ b/eng/docker-tools/templates/steps/clean-acr-images.yml @@ -0,0 +1,33 @@ +parameters: + repo: null + acr: null + action: null + age: null + customArgs: "--dry-run" + internalProjectName: null + publishConfig: null +steps: + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + # Options are documented in CleanAcrImagesOptions.cs + ${{ if eq(parameters.action, 'delete') }}: + displayName: "Delete ${{ parameters.repo }}" + ${{ elseif parameters.age }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }} > ${{ parameters.age }}d)" + ${{ else }}: + displayName: "Clean ${{ parameters.repo }} (${{ parameters.action }})" + serviceConnections: + - name: acr + id: ${{ parameters.publishConfig.cleanServiceConnection.id }} + tenantId: ${{ parameters.publishConfig.cleanServiceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.cleanServiceConnection.clientId }} + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + cleanAcrImages + ${{ parameters.repo }} + ${{ parameters.acr.subscription }} + ${{ parameters.acr.resourceGroup }} + ${{ parameters.acr.server }} + --action ${{ parameters.action }} + --age ${{ parameters.age }} + ${{ parameters.customArgs }} diff --git a/eng/docker-tools/templates/steps/cleanup-docker-linux.yml b/eng/docker-tools/templates/steps/cleanup-docker-linux.yml new file mode 100644 index 000000000..ddf0f9b09 --- /dev/null +++ b/eng/docker-tools/templates/steps/cleanup-docker-linux.yml @@ -0,0 +1,15 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup local Docker server + ################################################################################ +- script: docker stop $(docker ps -q) || true + displayName: Stop Running Containers + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- script: docker system prune -a -f --volumes + displayName: Cleanup Docker + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/cleanup-docker-windows.yml b/eng/docker-tools/templates/steps/cleanup-docker-windows.yml new file mode 100644 index 000000000..a2c41c20b --- /dev/null +++ b/eng/docker-tools/templates/steps/cleanup-docker-windows.yml @@ -0,0 +1,18 @@ +parameters: + condition: true + +steps: + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- powershell: $(engDockerToolsPath)/Invoke-CleanupDocker.ps1 + displayName: Cleanup Docker Images + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: | + if (Test-Path $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder) { + Remove-Item $(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder -Force -Recurse; + } + displayName: Cleanup Image Builder + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/copy-base-images.yml b/eng/docker-tools/templates/steps/copy-base-images.yml new file mode 100644 index 000000000..d54737083 --- /dev/null +++ b/eng/docker-tools/templates/steps/copy-base-images.yml @@ -0,0 +1,52 @@ +parameters: +- name: acr + type: object + default: + server: "" + serviceConnection: + tenantId: "" + clientId: "" + id: "" + subscription: "" + resourceGroup: "" +- name: repoPrefix + type: string + default: null +- name: additionalOptions + type: string + default: "" +- name: continueOnError + type: string + default: false +- name: forceDryRun + type: boolean + default: false + +steps: +- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Copy Base Images + serviceConnections: + - name: "acr" + tenantId: ${{ parameters.acr.serviceConnection.tenantId }} + clientId: ${{ parameters.acr.serviceConnection.clientId }} + id: ${{ parameters.acr.serviceConnection.id }} + continueOnError: ${{ parameters.continueOnError }} + internalProjectName: 'internal' + # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined, + # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an + # error + args: >- + copyBaseImages + '${{ parameters.acr.subscription }}' + '${{ parameters.acr.resourceGroup }}' + $(dockerHubRegistryCreds) + $(customCopyBaseImagesArgs) + --repo-prefix '${{ parameters.repoPrefix }}' + --registry-override '${{ parameters.acr.server }}' + --os-type 'linux' + --architecture '*' + $env:DRYRUNARG + ${{ parameters.additionalOptions }} diff --git a/eng/docker-tools/templates/steps/download-build-artifact.yml b/eng/docker-tools/templates/steps/download-build-artifact.yml new file mode 100644 index 000000000..965f49320 --- /dev/null +++ b/eng/docker-tools/templates/steps/download-build-artifact.yml @@ -0,0 +1,40 @@ +parameters: + # Destination directory on the pipeline agent's filesystem, relative or absolute. + targetPath: "" + # The build/pipeline artifact to download. If the value is left empty, + # the task downloads all artifacts associated with the pipeline run. + artifactName: "" + # AKA pipeline/definition - optional. + # If this is left empty, use the current pipeline's definition ID. + # You can get this from the URL of the pipeline's overview page on Azure DevOps. + # Example: https://dev.azure.com/$org/$project/_build?definitionId=373 + pipelineDefinitionId: "" + # AKA runId/buildId/pipelineId - optional. + # The identifier of the pipeline run from which to download the artifacts. + # If this is left empty, then always download from the current pipeline + # You can get this from the URL of the specific pipeline run, for example: + # https://dev.azure.com/$org/$project/_build/results?buildId=2709155&view=results + pipelineRunId: "" + condition: true + continueOnError: false + +steps: +# https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/download-pipeline-artifact-v2 +- task: DownloadPipelineArtifact@2 + inputs: + ${{ if ne(parameters.pipelineRunId, '') }}: + buildType: specific + project: $(System.TeamProject) + ${{ if ne(parameters.pipelineDefinitionId, '') }}: + definition: ${{ parameters.pipelineDefinitionId }} + ${{ else }}: + definition: $(System.DefinitionId) + buildId: ${{ parameters.pipelineRunId }} + buildVersionToDownload: specific + ${{ else }}: + buildType: current + targetPath: ${{ parameters.targetPath }} + artifactName: ${{ parameters.artifactName }} + displayName: Download Build Artifact(s) + condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml new file mode 100644 index 000000000..c22ad449c --- /dev/null +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -0,0 +1,9 @@ +parameters: + condition: true + +steps: +- powershell: | + $sourceBranch=$Env:BUILD_SOURCEBRANCH -replace "refs/heads/","" -replace "refs/tags/","" -replace "refs/pull/","" + echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch" + displayName: Define Source Branch Variable + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-docker-linux.yml b/eng/docker-tools/templates/steps/init-docker-linux.yml new file mode 100644 index 000000000..c8669e19b --- /dev/null +++ b/eng/docker-tools/templates/steps/init-docker-linux.yml @@ -0,0 +1,90 @@ +parameters: + setupImageBuilder: true + setupTestRunner: false + cleanupDocker: false + condition: true + +steps: +- template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + condition: ${{ parameters.condition }} +- script: echo "##vso[task.setvariable variable=artifactsPath]/artifacts" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) + + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- ${{ if eq(parameters.cleanupDocker, 'true') }}: + - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self + parameters: + condition: ${{ parameters.condition }} + + ################################################################################ + # Setup Image Builder (Optional) + ################################################################################ +- ${{ if eq(parameters.setupImageBuilder, 'true') }}: + + - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.imageBuilder) + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + + - script: >- + docker build + -t $(imageNames.imageBuilder.withrepo) + --build-arg IMAGE=$(imageNames.imageBuilder) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Image for Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + + - task: PowerShell@2 + displayName: Define ImageBuilder Command Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" + Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" + + $dockerRunBaseCmd = @( + "docker run --rm" + ) + + $dockerRunArgs = @( + "-v /var/run/docker.sock:/var/run/docker.sock" + "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" + "-w /repo" + "$(imageBuilderDockerRunExtraOptions)" + "$(imageNames.imageBuilder.withrepo)" + ) + + $authedDockerRunArgs = @( + '-e' + 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' + '-e' + 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' + ) + + $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs + $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + + $runImageBuilderCmd = $($dockerRunCmd -join ' ') + $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') + + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" + + ################################################################################ + # Setup Test Runner (Optional) + ################################################################################ +- ${{ if eq(parameters.setupTestRunner, 'true') }}: + - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.testrunner) + displayName: Pull Test Runner + condition: and(succeeded(), ${{ parameters.condition }}) + - script: > + docker build + -t $(imageNames.testRunner.withrepo) + --build-arg IMAGE=$(imageNames.testrunner) + -f $(engDockerToolsPath)/Dockerfile.WithRepo . + displayName: Build Test Runner Image + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-docker-windows.yml b/eng/docker-tools/templates/steps/init-docker-windows.yml new file mode 100644 index 000000000..e8593b5eb --- /dev/null +++ b/eng/docker-tools/templates/steps/init-docker-windows.yml @@ -0,0 +1,48 @@ +parameters: + setupImageBuilder: true + condition: true + +steps: +- template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + condition: ${{ parameters.condition }} +- powershell: echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" + displayName: Define Artifacts Path Variable + condition: and(succeeded(), ${{ parameters.condition }}) + + ################################################################################ + # Cleanup Docker Resources + ################################################################################ +- template: /eng/docker-tools/templates/steps/cleanup-docker-windows.yml@self + parameters: + condition: ${{ parameters.condition }} + + ################################################################################ + # Setup Image Builder (Optional) + ################################################################################ +- ${{ if eq(parameters.setupImageBuilder, 'true') }}: + - powershell: $(engDockerToolsPath)/Invoke-WithRetry.ps1 "docker pull $(imageNames.imageBuilder)" + displayName: Pull Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker create --name setupImageBuilder-$(Build.BuildId)-$(System.JobId) $(imageNames.imageBuilder) + displayName: Create Setup Container + condition: and(succeeded(), ${{ parameters.condition }}) + - script: > + docker cp + setupImageBuilder-$(Build.BuildId)-$(System.JobId):/image-builder + $(Build.BinariesDirectory)/.Microsoft.DotNet.ImageBuilder + displayName: Copy Image Builder + condition: and(succeeded(), ${{ parameters.condition }}) + - script: docker rm -f setupImageBuilder-$(Build.BuildId)-$(System.JobId) + displayName: Cleanup Setup Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + - task: PowerShell@2 + displayName: Define runImageBuilderCmd Variables + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + $runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe" + Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" + Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runImageBuilderCmd" diff --git a/eng/docker-tools/templates/steps/init-matrix-build-publish.yml b/eng/docker-tools/templates/steps/init-matrix-build-publish.yml new file mode 100644 index 000000000..8a555eccd --- /dev/null +++ b/eng/docker-tools/templates/steps/init-matrix-build-publish.yml @@ -0,0 +1,78 @@ +# Initialize common variables used in +# - Generating build matrix +# - Building images +# - Running tests +# - Publishing images + +parameters: + publishConfig: null + versionsRepoRef: "" + versionsRepoPath: "versions" + +steps: +- checkout: self +- ${{ if ne(parameters.versionsRepoRef, '') }}: + - checkout: ${{ parameters.versionsRepoRef }} + path: s/${{ parameters.versionsRepoPath }} + persistCredentials: true + fetchDepth: 1 + condition: succeeded() +- powershell: | + $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" + if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} --registry-override ${{ parameters.publishConfig.buildAcr.server }}" + } + + if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" + } + + if ("${{ parameters.versionsRepoRef }}" -ne "") { + $versionsBasePath = "${{ parameters.versionsRepoPath }}/" + $pipelineDisabledCache = "false" + + $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") + if ($pathSeparatorIndex -ge 0) { + $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) + } + else { + $buildRepoName = "$(Build.Repository.Name)" + } + + $engDockerToolsPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engDockerToolsRelativePath)" + + $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" + $manifest = "$buildRepoName/$(manifest)" + $testResultsDirectory = "$buildRepoName/$testResultsDirectory" + + if ("$(testScriptPath)") { + $testScriptPath = "$buildRepoName/$(testScriptPath)" + } + + echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" + echo "##vso[task.setvariable variable=engDockerToolsPath]$engDockerToolsPath" + echo "##vso[task.setvariable variable=manifest]$manifest" + echo "##vso[task.setvariable variable=engPath]$engPath" + echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" + echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" + } + else { + $versionsBasePath = "" + $pipelineDisabledCache = "true" + } + + echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" + echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" + echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" + displayName: Set Common Variables for Matrix, Build, and Publish + +- ${{ if ne(parameters.versionsRepoRef, '') }}: + # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. + # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the + # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. + - task: CopyFiles@2 + displayName: Copy TSA Config + inputs: + SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' + Contents: '.config/tsaoptions.json' + TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml b/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml new file mode 100644 index 000000000..5ab18214d --- /dev/null +++ b/eng/docker-tools/templates/steps/parse-test-arg-arrays.yml @@ -0,0 +1,15 @@ +steps: +- powershell: | + # Formats the OS versions in a compact human-readable form (e.g. "os1/os2") + $osVersionsDisplayName = '$(osVersions)' -Replace '--os-version ', '' -Replace ' ', '/' + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the OS versions + $osVersionsArrayInitStr = "@('" + $($osVersionsDisplayName -Replace "/", "', '") + "')" + + echo "##vso[task.setvariable variable=osVersionsDisplayName]$osVersionsDisplayName" + echo "##vso[task.setvariable variable=osVersionsArrayInitStr]$osVersionsArrayInitStr" + + # Defines a PowerShell snippet in string-form that can be used to initialize an array of the image builder paths + $pathInitStr = "@('" + $('$(imageBuilderPaths)' -Replace '--path', '' -Replace " ", "', '") + "')" + echo "##vso[task.setvariable variable=imageBuilderPathsArrayInitStr]$pathInitStr" + displayName: Parse Test Arg Arrays diff --git a/eng/docker-tools/templates/steps/publish-artifact.yml b/eng/docker-tools/templates/steps/publish-artifact.yml new file mode 100644 index 000000000..72ce47005 --- /dev/null +++ b/eng/docker-tools/templates/steps/publish-artifact.yml @@ -0,0 +1,28 @@ +parameters: +- name: path + type: string +- name: artifactName + type: string +- name: displayName + type: string +- name: internalProjectName + type: string +- name: publicProjectName + type: string +- name: condition + type: string + default: 'true' + +steps: +- ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + path: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + - publish: ${{ parameters.path }} + artifact: ${{ parameters.artifactName }} + displayName: ${{ parameters.displayName }} + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/publish-readmes.yml b/eng/docker-tools/templates/steps/publish-readmes.yml new file mode 100644 index 000000000..270898050 --- /dev/null +++ b/eng/docker-tools/templates/steps/publish-readmes.yml @@ -0,0 +1,29 @@ +parameters: + dryRunArg: "" + condition: true + +steps: +- script: > + $(runImageBuilderCmd) publishMcrDocs + --manifest '$(manifest)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' + '$(mcrDocsRepoInfo.userName)' + '$(mcrDocsRepoInfo.email)' + $(mcrDocsRepoInfo.authArgs) + '$(publicGitRepoUri)' + ${{ parameters.dryRunArg }} + $(manifestVariables) + $(imageBuilder.queueArgs) + --git-owner 'Microsoft' + --git-repo 'mcrdocs' + --git-branch 'main' + --git-path 'teams' + $(additionalPublishMcrDocsArgs) + name: PublishReadmes + displayName: Publish Readmes + condition: ${{ parameters.condition }} +- template: /eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml@self + parameters: + commitDigest: $(PublishReadmes.readmeCommitDigest) + condition: and(${{ parameters.condition }}, ne(variables['PublishReadmes.readmeCommitDigest'], '')) + dryRunArg: ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/steps/retain-build.yml b/eng/docker-tools/templates/steps/retain-build.yml new file mode 100644 index 000000000..bcf11d57e --- /dev/null +++ b/eng/docker-tools/templates/steps/retain-build.yml @@ -0,0 +1,9 @@ +steps: +- powershell: > + $(engDockerToolsPath)/Retain-Build.ps1 + -BuildId $(Build.BuildId) + -AzdoOrgUri '$(System.CollectionUri)' + -AzdoProject '$(System.TeamProject)' + -Token '$(System.AccessToken)' + displayName: Enable permanent build retention + condition: and(succeeded(), eq(variables.retainBuild, 'true')) diff --git a/eng/docker-tools/templates/steps/run-imagebuilder.yml b/eng/docker-tools/templates/steps/run-imagebuilder.yml new file mode 100644 index 000000000..3613d809b --- /dev/null +++ b/eng/docker-tools/templates/steps/run-imagebuilder.yml @@ -0,0 +1,76 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run ImageBuilder" +- name: serviceConnections + type: object + default: + # name: the name of the service connection argument that will be passed to the ImageBuilder command. + # For example, if the argument is --acr-service-connection, the name would be "acr". + - name: "" + # The service connection's ID (GUID). + id: "" + # The client ID of the Managed Idendity backing the service connection (GUID). + clientId: "" + # The ID of the tenant that the Managed Identity is in (GUID). + tenantId: "" +- name: internalProjectName + type: string + default: null +- name: args + type: string + default: null +- name: condition + type: string + default: succeeded() +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_OIDCREQUESTURI: $(System.OidcRequestUri) + inputs: + targetType: 'inline' + script: | + $serviceConnections = '${{ convertToJson(parameters.serviceConnections) }}' + + Write-Host "Service connections:" + Write-Host "${serviceConnections}" + + $serviceConnectionsJson = $serviceConnections | ConvertFrom-Json + $serviceConnectionsArgs = @() + foreach ($connection in $serviceConnectionsJson) { + $serviceConnectionsArgs += "--$($connection.name)-service-connection" + $serviceConnectionsArgs += "$($connection.tenantId):$($connection.clientId):$($connection.id)" + } + + $(runAuthedImageBuilderCmd) ${{ parameters.args }} @serviceConnectionsArgs + +- ${{ else }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + inputs: + targetType: 'inline' + script: >- + $(runImageBuilderCmd) ${{ parameters.args }} diff --git a/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml b/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml new file mode 100644 index 000000000..84de1e1f5 --- /dev/null +++ b/eng/docker-tools/templates/steps/run-pwsh-with-auth.yml @@ -0,0 +1,39 @@ +parameters: +- name: name + type: string + default: "" +- name: displayName + type: string + default: "Run PowerShell" +- name: serviceConnection + type: string + default: "" +- name: command + type: string + default: null +- name: continueOnError + type: boolean + default: false +- name: dockerClientOS + type: string + default: "linux" +- name: condition + type: string + default: true + +steps: +- task: AzureCLI@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} (Authenticated) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + addSpnToEnvironment: true + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: 'ps' + ${{ else }}: + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: ${{ parameters.command }}; diff --git a/eng/docker-tools/templates/steps/set-dry-run.yml b/eng/docker-tools/templates/steps/set-dry-run.yml new file mode 100644 index 000000000..44e3062b0 --- /dev/null +++ b/eng/docker-tools/templates/steps/set-dry-run.yml @@ -0,0 +1,30 @@ +parameters: + name: publishConfig + type: object + +steps: +- powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + # Don't use dry-run mode for unofficial builds, since they publish to a + # non-production environment + $dryRunArg="" + } + elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") + { + # Public builds need to use dry-run mode since they don't publish anywhere. + $dryRunArg="--dry-run" + } + elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) + { + # If we're running an internal build on an official pipeline but not + # publishing to an official repo prefix, then use dry run mode. + $dryRunArg="--dry-run" + } + else + { + $dryRunArg="" + } + + echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" + displayName: Set dry-run arg for non-prod diff --git a/eng/docker-tools/templates/steps/set-image-info-path-var.yml b/eng/docker-tools/templates/steps/set-image-info-path-var.yml new file mode 100644 index 000000000..71d317fa2 --- /dev/null +++ b/eng/docker-tools/templates/steps/set-image-info-path-var.yml @@ -0,0 +1,19 @@ +parameters: + publicSourceBranch: null + +steps: +- powershell: | + $basePath = "$(gitHubVersionsRepoInfo.path)" + + $publicSourceBranch = "${{ parameters.publicSourceBranch }}" + + if ($publicSourceBranch -eq "") { + throw "publicSourceBranch variable is not set" + } + + $buildRepoName = "$(Build.Repository.Name)".Replace("/", "-") + $imageInfoName = "image-info.$buildRepoName-$publicSourceBranch$(imageInfoVariant).json" + + echo "##vso[task.setvariable variable=imageInfoVersionsPath]$basePath/$imageInfoName" + echo "##vso[task.setvariable variable=gitHubImageInfoVersionsPath]$(gitHubVersionsRepoInfo.path)/$imageInfoName" + displayName: Set Image Info Path Vars diff --git a/eng/docker-tools/templates/steps/test-images-linux-client.yml b/eng/docker-tools/templates/steps/test-images-linux-client.yml new file mode 100644 index 000000000..38426ab2a --- /dev/null +++ b/eng/docker-tools/templates/steps/test-images-linux-client.yml @@ -0,0 +1,107 @@ +parameters: + preBuildValidation: false + internalProjectName: null + publishConfig: null + condition: true + customInitSteps: [] + sourceBuildPipelineRunId: "" + +steps: +- template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + parameters: + setupImageBuilder: false + setupTestRunner: true + # Clean only up when we're running an internal build, not a PR, and not doing pre-build validation. + # i.e. when we're building something important. + cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} + condition: ${{ parameters.condition }} +- ${{ parameters.customInitSteps }} +- script: | + echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" + + additionalTestArgs="$ADDITIONALTESTARGS" + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories pre-build" + else + if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then + additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" + fi + fi + fi + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- script: > + docker run -t -d + -v /var/run/docker.sock:/var/run/docker.sock + -v $(Build.ArtifactStagingDirectory):$(artifactsPath) + -e DOCKER_BUILDKIT=1 + -e RUNNING_TESTS_IN_CONTAINER=true + --name $(testRunner.container) + $(imageNames.testRunner.withrepo) + displayName: Start Test Runner Container + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; + docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh + -File $(engDockerToolsRelativePath)/Invoke-WithRetry.ps1 + "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.buildAcr.server }}" + - ${{ if eq(parameters.preBuildValidation, 'false') }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + docker exec + $(testRunner.options) + $(testRunner.container) + pwsh + -Command "$(testScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + -Architecture '$(architecture)' + $(additionalTestArgs)" + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.buildAcr.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- powershell: > + docker cp + $(testRunner.container):/repo/$(testResultsDirectory) + $(Common.TestResultsDirectory)/. + displayName: Copy Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Common.TestResultsDirectory) + mergeTestResults: true + publishRunAttachments: true + ${{ if eq(parameters.preBuildValidation, 'false') }}: + testRunTitle: $(productVersion) $(osVersionsDisplayName) $(architecture) + ${{ if eq(parameters.preBuildValidation, 'true') }}: + testRunTitle: Pre-Build Validation +- script: docker rm -f $(testRunner.container) + displayName: Cleanup TestRunner Container + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true diff --git a/eng/docker-tools/templates/steps/test-images-windows-client.yml b/eng/docker-tools/templates/steps/test-images-windows-client.yml new file mode 100644 index 000000000..ad84cff0e --- /dev/null +++ b/eng/docker-tools/templates/steps/test-images-windows-client.yml @@ -0,0 +1,66 @@ +parameters: + internalProjectName: null + publishConfig: null + condition: true + customInitSteps: [] + sourceBuildPipelineRunId: "" + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/init-docker-windows.yml@self + parameters: + cleanupDocker: true + setupImageBuilder: false + condition: ${{ parameters.condition }} + - template: /eng/docker-tools/templates/steps/run-pwsh-with-auth.yml@self + parameters: + displayName: Docker login + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} + dockerClientOS: windows + condition: and(succeeded(), ${{ parameters.condition }}) + command: >- + az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; + $accessToken = $(az acr login -n ${{ parameters.publishConfig.buildAcr.server }} --expose-token --query accessToken --output tsv); + docker login ${{ parameters.publishConfig.buildAcr.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken +- ${{ parameters.customInitSteps }} +- powershell: | + if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" + } + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" + displayName: Set Test Variables + condition: and(succeeded(), ${{ parameters.condition }}) +- powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item + displayName: Cleanup Old Test Results + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: image-info + condition: ${{ parameters.condition }} + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} +- template: /eng/docker-tools/templates/steps/parse-test-arg-arrays.yml@self +- powershell: > + $(test.init); + $(testScriptPath) + -Paths $(imageBuilderPathsArrayInitStr) + -OSVersions $(osVersionsArrayInitStr) + $(additionalTestArgs) + displayName: Test Images + condition: and(succeeded(), ${{ parameters.condition }}) +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker logout ${{ parameters.publishConfig.buildAcr.server }} + displayName: Docker logout + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: and(always(), ${{ parameters.condition }}) + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '$(testResultsDirectory)/**/*.trx' + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(productVersion) $(osVersionsDisplayName) amd64 diff --git a/eng/docker-tools/templates/steps/validate-branch.yml b/eng/docker-tools/templates/steps/validate-branch.yml new file mode 100644 index 000000000..0fb1a841d --- /dev/null +++ b/eng/docker-tools/templates/steps/validate-branch.yml @@ -0,0 +1,52 @@ +parameters: + publishConfig: null + internalProjectName: null + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + echo "Build is from an unofficial pipeline, continuing." + exit 0 + } + + $isOfficialRepoPrefix = "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}") + if (-not $isOfficialRepoPrefix) + { + echo "This build will not publish to an official repo prefix, continuing." + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" + echo "Official repo prefixes: $(officialRepoPrefixes)" + exit 0 + } + + $isOfficialBranch = "$(officialBranches)".Split(',').Contains("$(sourceBranch)") + if ($isOfficialBranch) + { + echo "$(sourceBranch) is an official branch, continuing." + echo "Official branches: $(officialBranches)" + exit 0 + } + + $hasOfficialBranchPrefix = $false + foreach ($prefix in "$(officialBranchPrefixes)".Split(',')) { + if ("$(sourceBranch)".StartsWith($prefix)) { + $hasOfficialBranchPrefix = $true + break + } + } + + if ($hasOfficialBranchPrefix) + { + echo "$(sourceBranch) has an official branch prefix, continuing." + echo "Official branch prefixes: $(officialBranchPrefixes)" + exit 0 + } + + echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." + echo "Build definition: $(Build.DefinitionName)" + echo "1ESPT build type: $(OneESPT.BuildType)" + echo "Current branch: $(sourceBranch)" + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" + exit 1 + displayName: Validate Branch diff --git a/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml b/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml new file mode 100644 index 000000000..5f64438bb --- /dev/null +++ b/eng/docker-tools/templates/steps/wait-for-mcr-doc-ingestion.yml @@ -0,0 +1,21 @@ +parameters: + commitDigest: null + condition: true + dryRunArg: "" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for MCR Doc Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrDocIngestion + '${{ parameters.commitDigest }}' + --timeout '$(mcrDocIngestionTimeout)' + ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml new file mode 100644 index 000000000..6c7236439 --- /dev/null +++ b/eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml @@ -0,0 +1,37 @@ +parameters: +- name: publishConfig + type: object + +- name: imageInfoPath + type: string + +- name: minQueueTime + type: string + +- name: dryRunArg + type: string + +- name: condition + type: string + default: "true" + +steps: +- template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Image Ingestion + condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) + internalProjectName: 'internal' + args: >- + waitForMcrImageIngestion + '${{ parameters.imageInfoPath }}' + --manifest '$(manifest)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' + --min-queue-time '${{ parameters.minQueueTime }}' + --timeout '$(mcrImageIngestionTimeout)' + $(manifestVariables) + ${{ parameters.dryRunArg }} diff --git a/eng/docker-tools/templates/task-prefix-decorator.yml b/eng/docker-tools/templates/task-prefix-decorator.yml new file mode 100644 index 000000000..598bfbe24 --- /dev/null +++ b/eng/docker-tools/templates/task-prefix-decorator.yml @@ -0,0 +1,63 @@ +# This Azure Pipelines template adds a prefix to the display name of each +# task passed through the `stages` parameter. When used in conjunction with +# an "extends" template which injects a lot of tasks into the pipeline, the +# added prefix helps to identify which tasks were passed through this template +# and which tasks were injected by the `baseTemplate`. +# +# This template assumes that `baseTemplate` uses the `stages` parameter. If it +# doesn't, this template likely won't work as expected. + +parameters: +# The pipeline will behave as if it were originally extended from this template, +# except with updated task display names. +- name: baseTemplate + type: string + default: "" + +# These parameters are passed directly to `baseTemplate` +- name: templateParameters + type: object + default: null + +# These stages will be modified and passed to the `baseTemplate` as the +# `stages` parameter. +- name: stages + type: stageList + default: [] + +# This prefix will be added to the display name of each task. +- name: taskPrefix + type: string + default: "🟪" + + +extends: + template: ${{ parameters.baseTemplate }} + parameters: + ${{ insert }}: ${{ parameters.templateParameters }} + stages: + - ${{ each stage in parameters.stages }}: + - stage: ${{ stage.stage }} + ${{ each property in stage }}: + ${{ if notIn(property.key, 'stage', 'jobs') }}: + ${{ property.key }} : ${{ property.value }} + jobs: + - ${{ each job in stage.jobs }}: + - job: ${{ job.job }} + ${{ each property in job }}: + ${{ if notIn(property.key, 'job', 'steps') }}: + ${{ property.key }} : ${{ property.value }} + steps: + - ${{ each step in job.steps }}: + # Special case for Azure Pipelines checkout task: + # https://learn.microsoft.com/azure/devops/extend/develop/pipeline-decorator-context?view=azure-devops#task-names-and-guids + # The checkout task does not have a name - it is special and built directly into the agent. + # Avoid modifying the checkout task, or else it will show up in the UI as a task with no name. + - ${{ if contains(step.task, '6d15af64-176c-496d-b583-fd2ae21d4df4') }}: + - ${{ step }} + - ${{ else }}: + - task: ${{ step.task }} + ${{ each property in step }}: + ${{ if notIn(property.key, 'task', 'displayName') }}: + ${{ property.key }} : ${{ property.value }} + displayName: ${{ parameters.taskPrefix }} ${{ step.displayName }} diff --git a/eng/docker-tools/templates/variables/common-paths.yml b/eng/docker-tools/templates/variables/common-paths.yml new file mode 100644 index 000000000..d8a520025 --- /dev/null +++ b/eng/docker-tools/templates/variables/common-paths.yml @@ -0,0 +1,5 @@ +variables: + engDockerToolsRelativePath: eng/docker-tools + engDockerToolsPath: $(Build.Repository.LocalPath)/$(engDockerToolsRelativePath) + engPath: $(Build.Repository.LocalPath)/eng + testScriptPath: "" diff --git a/eng/docker-tools/templates/variables/common.yml b/eng/docker-tools/templates/variables/common.yml new file mode 100644 index 000000000..cef4424c1 --- /dev/null +++ b/eng/docker-tools/templates/variables/common.yml @@ -0,0 +1,82 @@ +variables: +- template: /eng/docker-tools/templates/variables/docker-images.yml@self +- template: /eng/docker-tools/templates/variables/common-paths.yml@self + +- name: publishReadme + value: true +- name: publishImageInfo + value: true +- name: ingestKustoImageInfo + value: true + # CG is disabled by default because projects are built within Dockerfiles and CG step do not scan artifacts + # that are built within Dockerfiles. A separate CG pipeline exists for this reason. +- name: skipComponentGovernanceDetection + value: false +- name: build.imageBuilderDockerRunExtraOptions + value: "" +- name: imageBuilderDockerRunExtraOptions + value: "" +- name: generateEolAnnotationDataExtraOptions + value: "" +- name: productVersionComponents + value: 2 +- name: imageInfoVariant + value: "" +- name: publishNotificationsEnabled + value: false +- name: manifestVariables + value: "" +- name: mcrImageIngestionTimeout + value: "00:20:00" +- name: mcrDocIngestionTimeout + value: "00:05:00" +- name: officialBranches + # comma-delimited list of branch names + value: main +- name: mirrorRepoPrefix + value: 'mirror/' +- name: cgBuildGrepArgs + value: "''" +- name: test.init + value: "" +- name: testRunner.options + value: "" +- name: customCopyBaseImagesArgs + value: "" +- name: additionalGenerateBuildMatrixOptions + value: "" +- name: trimCachedImagesForMatrix + value: false + +- name: defaultLinuxAmd64PoolImage + value: ubuntu-latest +- name: defaultLinuxArm32PoolImage + value: null +- name: defaultLinuxArm64PoolImage + value: null +- name: defaultWindows2016PoolImage + value: vs2017-win2016 +- name: defaultWindows1809PoolImage + value: windows-2019 +- name: defaultWindows2022PoolImage + value: windows-2022 +- name: defaultWindows2025PoolImage + value: windows-2025 + +- name: default1ESInternalPoolName + value: NetCore1ESPool-Internal +- name: default1ESInternalPoolImage + value: 1es-ubuntu-2204 + +- name: defaultSourceAnalysisPoolName + value: NetCore1ESPool-Internal +- name: defaultSourceAnalysisPoolImage + value: 1es-windows-2022 + +# Define these as placeholder values to allow string validation to succeed since we don't have the +# variable group with the actual values in public builds. For internal builds, the variable group +# will cause these values to be overridden with the real values. +- name: acr.subscription + value: 00000000-0000-0000-0000-000000000000 +- name: acr-staging.subscription + value: 00000000-0000-0000-0000-000000000000 diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml new file mode 100644 index 000000000..80c281a7e --- /dev/null +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -0,0 +1,7 @@ +variables: + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2843774 + imageNames.imageBuilder: $(imageNames.imageBuilderName) + imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner + imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.syft: anchore/syft:v1.31.0-debug diff --git a/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml b/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml new file mode 100644 index 000000000..96a0cf500 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/build-test-publish.yml @@ -0,0 +1,48 @@ +# Common variables for building/testing/publishing in the .NET team's pipelines + +variables: +- template: /eng/docker-tools/templates/variables/dotnet/common.yml@self + +- name: commonVersionsImageInfoPath + value: build-info/docker +- name: publicGitRepoUri + value: https://github.com/dotnet/dotnet-docker +- name: testScriptPath + value: ./tests/run-tests.ps1 +- name: testResultsDirectory + value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ + +- name: officialRepoPrefixes + value: public/,internal/private/,unlisted/ + readonly: true + +- name: mcrDocsRepoInfo.userName + value: $(gitHubApp.marDocsUpdater.userName) +- name: mcrDocsRepoInfo.email + value: $(gitHubApp.marDocsUpdater.email) + +- name: publishNotificationsEnabled + value: true +- name: gitHubNotificationsRepoInfo.org + value: dotnet +- name: gitHubNotificationsRepoInfo.repo + value: dotnet-docker-internal +# $(gitHubNotificationsRepoInfo.authArgs) is needed by the "Post Publish +# Notification" step in eng/docker-tools/templates/jobs/publish.yml#L271, even during +# a dry-run. This value is a placeholder that gets replaced when referencing +# the secrets.yml variable template. +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token 'placeholder' + +- name: gitHubVersionsRepoInfo.org + value: dotnet +- name: gitHubVersionsRepoInfo.repo + value: versions +- name: gitHubVersionsRepoInfo.branch + value: main +- name: gitHubVersionsRepoInfo.path + value: ${{ variables.commonVersionsImageInfoPath }} +- name: gitHubVersionsRepoInfo.userName + value: $(dotnetDockerBot.userName) +- name: gitHubVersionsRepoInfo.email + value: $(dotnetDockerBot.email) diff --git a/eng/docker-tools/templates/variables/dotnet/common.yml b/eng/docker-tools/templates/variables/dotnet/common.yml new file mode 100644 index 000000000..8f98c0921 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/common.yml @@ -0,0 +1,63 @@ +variables: +- template: /eng/docker-tools/templates/variables/common.yml@self + +- name: publicProjectName + value: public +- name: internalProjectName + value: internal + +# $(dockerHubRegistryCreds) is needed by the copy-base-images step in +# eng/docker-tools/templates/stages/build-and-test.yml#L73-L78, even during a dry-run. +# This is a placeholder that gets replaced when referencing the secrets.yml +# variable template. +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=placeholder;placeholder' + +- name: linuxAmd64InternalPoolImage + value: 1es-ubuntu-2204 +- name: linuxAmd64InternalPoolName + value: NetCore1ESPool-Internal + +- name: linuxArm64PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm64PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm64InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: linuxArm32PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm32PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm32InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: windowsServer2016PublicPoolImage + value: Server2016-NESDockerBuilds +- name: windowsServer2016InternalPoolImage + value: Server2016-NESDockerBuilds-1ESPT +- name: windowsServer2016PoolName + value: Docker-2016-${{ variables['System.TeamProject'] }} + +- name: windowsServer2019PublicPoolImage + value: Server2019-1809-NESDockerBuilds +- name: windowsServer2019InternalPoolImage + value: Server2019-1809-NESDockerBuilds-1ESPT +- name: windowsServer2019PoolName + value: Docker-1809-${{ variables['System.TeamProject'] }} + +- name: windowsServer2022PublicPoolImage + value: Server2022-NESDockerBuilds +- name: windowsServer2022InternalPoolImage + value: Server2022-NESDockerBuilds-1ESPT +- name: windowsServer2022PoolName + value: Docker-2022-${{ variables['System.TeamProject'] }} + +- name: windowsServer2025PublicPoolImage + value: Server2025-NESDockerBuilds +- name: windowsServer2025InternalPoolImage + value: Server2025-NESDockerBuilds-1ESPT +- name: windowsServer2025PoolName + value: Docker-2025-${{ variables['System.TeamProject'] }} + +- group: DotNet-Docker-Common-2 diff --git a/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml b/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml new file mode 100644 index 000000000..1744ad288 --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/secrets-unofficial.yml @@ -0,0 +1,5 @@ +variables: +- group: DotNet-Docker-Secrets-Low + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnet-dockerhub-bot-username);$(dotnet-dockerhub-bot-pat-low)' diff --git a/eng/docker-tools/templates/variables/dotnet/secrets.yml b/eng/docker-tools/templates/variables/dotnet/secrets.yml new file mode 100644 index 000000000..0224c441e --- /dev/null +++ b/eng/docker-tools/templates/variables/dotnet/secrets.yml @@ -0,0 +1,17 @@ +variables: +- group: DotNet-Docker-Secrets + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' + +- name: gitHubNotificationsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: gitHubVersionsRepoInfo.authArgs + value: --gh-token '$(BotAccount-dotnet-docker-bot-PAT)' + +- name: mcrDocsRepoInfo.authArgs + value: >- + --gh-private-key '$(GitHubApp-NET-Docker-MAR-Docs-Updater-PrivateKey)' + --gh-app-client-id '$(gitHubApp.marDocsUpdater.clientId)' + --gh-app-installation-id '$(gitHubApp.marDocsUpdater.microsoft.installationId)'