[main] Update common Docker engineering infrastructure with latest

eng/docker-tools/Dockerfile.WithRepo

@@ -0,0 +1,6 @@
+# Use this Dockerfile to create an ImageBuilder image
+ARG IMAGE
+FROM $IMAGE
+
+WORKDIR /repo
+COPY . .

eng/docker-tools/Dockerfile.syft

@@ -0,0 +1,16 @@
+ARG SYFT_IMAGE_NAME
+ARG TARGET_IMAGE_NAME
+
+FROM ${SYFT_IMAGE_NAME} AS syft
+FROM ${TARGET_IMAGE_NAME} AS scan-image
+
+FROM syft AS run-scan
+ARG TARGET_IMAGE_NAME
+ENV SYFT_CHECK_FOR_APP_UPDATE=0 \
+    SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME}
+USER root
+RUN --mount=from=scan-image,source=/,target=/rootfs \
+    ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"]
+
+FROM scratch AS output
+COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json

eng/docker-tools/Get-BaseImageStatus.ps1

@@ -0,0 +1,33 @@
+#!/usr/bin/env pwsh
+
+<#
+.SYNOPSIS
+Outputs the status of external base images referenced in the Dockerfiles.
+#>
+[cmdletbinding()]
+param(
+    # Path to the manifest file to use
+    [string]
+    $Manifest = "manifest.json",
+
+    # Architecture to filter Dockerfiles to
+    [string]
+    $Architecture = "*",
+
+    # A value indicating whether to run the script continously
+    [switch]
+    $Continuous,
+
+    # Number of seconds to wait between each iteration
+    [int]
+    $ContinuousDelay = 10
+)
+
+Set-StrictMode -Version Latest
+
+$imageBuilderArgs = "getBaseImageStatus --manifest $Manifest --architecture $Architecture"
+if ($Continuous) {
+    $imageBuilderArgs += " --continuous --continuous-delay $ContinuousDelay"
+}
+
+& "$PSScriptRoot/Invoke-ImageBuilder.ps1" -ImageBuilderArgs $imageBuilderArgs

eng/docker-tools/Get-ImageBuilder.ps1

@@ -0,0 +1,13 @@
+#!/usr/bin/env pwsh
+
+# Load common image names
+$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1
+foreach ($varName in $imageNameVars.Keys) { 
+    Set-Variable -Name $varName -Value $imageNameVars[$varName] -Scope Global
+}
+
+& docker inspect ${imageNames.imagebuilderName} | Out-Null
+if (-not $?) {
+    Write-Output "Pulling"
+    & $PSScriptRoot/Invoke-WithRetry.ps1 "docker pull ${imageNames.imagebuilderName}"
+}

eng/docker-tools/Get-ImageNameVars.ps1

@@ -0,0 +1,12 @@
+# Returns a hashtable of variable name-to-value mapping representing the image name variables
+# used by the common build infrastructure.
+
+$vars = @{}
+Get-Content $PSScriptRoot/templates/variables/docker-images.yml |
+    Where-Object { $_.Trim().Length -gt 0 -and $_.Trim() -notlike 'variables:' -and $_.Trim() -notlike '# *' } |
+    ForEach-Object {
+        $parts = $_.Split(':', 2)
+        $vars[$parts[0].Trim()] = $parts[1].Trim()
+    }
+
+return $vars

eng/docker-tools/Install-DotNetSdk.ps1

@@ -0,0 +1,64 @@
+#!/usr/bin/env pwsh
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+<#
+.SYNOPSIS
+Install the .NET Core SDK at the specified path.
+
+.PARAMETER InstallPath
+The path where the .NET Core SDK is to be installed.
+
+.PARAMETER Channel
+The version of the .NET Core SDK to be installed.
+
+#>
+[cmdletbinding()]
+param(
+    [string]
+    $InstallPath,
+    [string]
+    $Channel = "9.0"
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+if (!(Test-Path "$InstallPath")) {
+    mkdir "$InstallPath" | Out-Null
+}
+
+$IsRunningOnUnix = $PSVersionTable.contains("Platform") -and $PSVersionTable.Platform -eq "Unix"
+if ($IsRunningOnUnix) {
+    $DotnetInstallScript = "dotnet-install.sh"
+}
+else {
+    $DotnetInstallScript = "dotnet-install.ps1"
+}
+
+$DotnetInstallScriptPath = Join-Path -Path $InstallPath -ChildPath $DotnetInstallScript
+
+if (!(Test-Path $DotnetInstallScriptPath)) {
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+    & "$PSScriptRoot/Invoke-WithRetry.ps1" "Invoke-WebRequest 'https://builds.dotnet.microsoft.com/dotnet/scripts/v1/$DotnetInstallScript' -OutFile $DotnetInstallScriptPath"
+}
+
+$DotnetChannel = $Channel
+
+$InstallFailed = $false
+if ($IsRunningOnUnix) {
+    & chmod +x $DotnetInstallScriptPath
+    & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath --channel $DotnetChannel --install-dir $InstallPath" -Retries 5
+    $InstallFailed = ($LASTEXITCODE -ne 0)
+}
+else {
+    & "$PSScriptRoot/Invoke-WithRetry.ps1" "$DotnetInstallScriptPath -Channel $DotnetChannel -InstallDir $InstallPath" -Retries 5
+    $InstallFailed = (-not $?)
+}
+
+# See https://github.com/NuGet/NuGet.Client/pull/4259
+$Env:NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY = "6,1500"
+
+if ($InstallFailed) { throw "Failed to install the .NET Core SDK" }

eng/docker-tools/Invoke-CleanupDocker.ps1

@@ -0,0 +1,20 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+docker ps -a -q | ForEach-Object { docker rm -f $_ }
+
+docker volume prune -f
+
+# Preserve the tagged Windows base images and the common eng infra images (e.g. ImageBuilder)
+# to avoid the expense of having to repull continuously.
+$imageNameVars = & $PSScriptRoot/Get-ImageNameVars.ps1
+
+docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" |
+    Where-Object {
+        $localImage = $_
+        $localImage.Contains(":<none> ")`
+        -Or -Not ($localImage.StartsWith("mcr.microsoft.com/windows")`
+            -Or ($imageNameVars.Values.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } |
+    ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } |
+    Select-Object -Unique |
+    ForEach-Object { docker rmi -f $_ }

eng/docker-tools/Invoke-ImageBuilder.ps1

@@ -0,0 +1,105 @@
+#!/usr/bin/env pwsh
+
+<#
+.SYNOPSIS
+Executes ImageBuilder with the specified args.
+
+.PARAMETER ImageBuilderArgs
+The args to pass to ImageBuilder.
+
+.PARAMETER ReuseImageBuilderImage
+Indicates that a previously built ImageBuilder image is presumed to exist locally and that
+it should be used for this execution of the script.  This allows some optimization when
+multiple calls are being made to this script that don't require a fresh image (i.e. the
+repo contents in the image don't need to be or should not be updated with each call to
+this script).
+
+.PARAMETER OnCommandExecuted
+A ScriptBlock that will be invoked after the ImageBuilder command has been executed.
+This allows the caller to execute extra logic in the context of the ImageBuilder while
+its container is still running.
+The ScriptBlock is passed the following argument values:
+    1. Container name
+#>
+[cmdletbinding()]
+param(
+    [string]
+    $ImageBuilderArgs,
+
+    [switch]
+    $ReuseImageBuilderImage,
+
+    [scriptblock]
+    $OnCommandExecuted
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Log {
+    param ([string] $Message)
+
+    Write-Output $Message
+}
+
+function Exec {
+    param ([string] $Cmd)
+
+    Log "Executing: '$Cmd'"
+    Invoke-Expression $Cmd
+    if ($LASTEXITCODE -ne 0) {
+        $host.SetShouldExit($LASTEXITCODE)
+        exit $LASTEXITCODE
+        throw "Failed: '$Cmd'"
+    }
+}
+
+$imageBuilderContainerName = "ImageBuilder-$(Get-Date -Format yyyyMMddhhmmss)"
+$containerCreated = $false
+
+pushd $PSScriptRoot/../../
+try {
+    $activeOS = docker version -f "{{ .Server.Os }}"
+    if ($activeOS -eq "linux") {
+        # On Linux, ImageBuilder is run within a container.
+        $imageBuilderImageName = "microsoft-dotnet-imagebuilder-withrepo"
+        if ($ReuseImageBuilderImage -ne $True) {
+            & ./eng/docker-tools/Get-ImageBuilder.ps1
+            Exec ("docker build -t $imageBuilderImageName --build-arg " `
+                + "IMAGE=${imageNames.imageBuilderName} -f eng/docker-tools/Dockerfile.WithRepo .")
+        }
+
+        $imageBuilderCmd = "docker run --name $imageBuilderContainerName -v /var/run/docker.sock:/var/run/docker.sock $imageBuilderImageName"
+        $containerCreated = $true
+    }
+    else {
+        # On Windows, ImageBuilder is run locally due to limitations with running Docker client within a container.
+        # Remove when https://github.com/dotnet/docker-tools/issues/159 is resolved
+        $imageBuilderFolder = ".Microsoft.DotNet.ImageBuilder"
+        $imageBuilderCmd = [System.IO.Path]::Combine($imageBuilderFolder, "Microsoft.DotNet.ImageBuilder.exe")
+        if (-not (Test-Path -Path "$imageBuilderCmd" -PathType Leaf)) {
+            & ./eng/docker-tools/Get-ImageBuilder.ps1
+            Exec "docker create --name $imageBuilderContainerName ${imageNames.imageBuilderName}"
+            $containerCreated = $true
+            if (Test-Path -Path $imageBuilderFolder)
+            {
+                Remove-Item -Recurse -Force -Path $imageBuilderFolder
+            }
+
+            Exec "docker cp ${imageBuilderContainerName}:/image-builder $imageBuilderFolder"
+        }
+    }
+
+    Exec "$imageBuilderCmd $ImageBuilderArgs"
+
+    if ($OnCommandExecuted) {
+        Invoke-Command $OnCommandExecuted -ArgumentList $imageBuilderContainerName
+    }
+}
+finally {
+    if ($containerCreated) {
+        Exec "docker container rm -f $imageBuilderContainerName"
+    }
+
+    popd
+}

eng/docker-tools/Invoke-WithRetry.ps1

@@ -0,0 +1,41 @@
+#!/usr/bin/env pwsh
+
+# Executes a command and retries if it fails.
+[cmdletbinding()]
+param (
+        [Parameter(Mandatory = $true)][string]$Cmd,
+        [int]$Retries = 2,
+        [int]$WaitFactor = 6
+    )
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$count = 0
+$completed = $false
+
+Write-Output "Executing '$Cmd'"
+
+while (-not $completed) {
+    try {
+        Invoke-Expression $Cmd
+        if (-not $(Test-Path variable:LASTEXITCODE) -or $LASTEXITCODE -eq 0) {
+            $completed = $true
+            continue
+        }
+    }
+    catch {
+    }
+
+    $count++
+
+    if ($count -lt $Retries) {
+        $wait = [Math]::Pow($WaitFactor, $count - 1)
+        Write-Output "Retry $count/$Retries, retrying in $wait seconds..."
+        Start-Sleep $wait
+    }
+    else {
+        Write-Output "Retry $count/$Retries, no more retries left."
+        throw "Failed to execute '$Cmd'"
+    }
+}

eng/docker-tools/Pull-Image.ps1

@@ -0,0 +1,18 @@
+#!/usr/bin/env pwsh
+
+[cmdletbinding()]
+param(
+    [Parameter(Mandatory = $true, Position = 0)]
+    [string]$Image,
+
+    [Parameter(Mandatory = $false)]
+    [int]$Retries = 2,
+
+    [Parameter(Mandatory = $false)]
+    [int]$WaitFactor = 6
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor

eng/docker-tools/Retain-Build.ps1

@@ -0,0 +1,43 @@
+# Adapted from https://github.com/dotnet/arcade/blob/main/eng/docker-tools/retain-build.ps1
+Param(
+    [Parameter(Mandatory = $true)][int] $BuildId,
+    [Parameter(Mandatory = $true)][string] $AzdoOrgUri,
+    [Parameter(Mandatory = $true)][string] $AzdoProject,
+    [Parameter(Mandatory = $true)][string] $Token
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+function Get-AzDOHeaders(
+    [string] $Token) {
+    $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${Token}"))
+    $headers = @{"Authorization" = "Basic $base64AuthInfo" }
+    return $headers
+}
+
+function Update-BuildRetention(
+    [string] $AzdoOrgUri,
+    [string] $AzdoProject,
+    [int] $BuildId,
+    [string] $Token) {
+    $headers = Get-AzDOHeaders -Token $Token
+    $requestBody = "{
+        `"keepForever`": `"true`"
+    }"
+
+    $requestUri = "${AzdoOrgUri}/${AzdoProject}/_apis/build/builds/${BuildId}?api-version=6.0"
+    Write-Host "Attempting to retain build using the following URI: ${requestUri} ..."
+
+    try {
+        Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json"
+        Write-Host "Updated retention settings for build ${BuildId}."
+    }
+    catch {
+        Write-Host "##[error] Failed to update retention settings for build: $($_.Exception.Response.StatusDescription)"
+        exit 1
+    }
+}
+
+Update-BuildRetention -AzdoOrgUri $AzdoOrgUri -AzdoProject $AzdoProject -BuildId $BuildId -Token $Token
+exit 0