Merge pull request #977 from microsoft/validate/sbom

Build SBOM manifest as part of build

azure-pipelines/artifacts/VSInsertion.ps1

@@ -12,6 +12,10 @@ if ($env:BUILDCONFIGURATION) { $config = $env:BUILDCONFIGURATION }
 $NuGetPackages = "$RepoRoot\bin\Packages\$config\NuGet"
 $CoreXTPackages = "$RepoRoot\bin\Packages\$config\CoreXT"
 if (-not (Test-Path $NuGetPackages)) { Write-Warning "No NuGet packages found. Has a build been run?"; return @{} }
+
+# This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet.
+if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $NuGetPackages/_manifest)) { return @{} }
+
 $ArtifactBasePath = "$RepoRoot\obj\_artifacts"
 $ArtifactPath = "$ArtifactBasePath\VSInsertion"
 if (-not (Test-Path $ArtifactPath)) { New-Item -ItemType Directory -Path $ArtifactPath | Out-Null }
@@ -36,6 +40,6 @@ if ($LASTEXITCODE -ne 0) {
 }
 
 @{
-    "$NuGetPackages" = (Get-ChildItem "$NuGetPackages\*.nupkg");
+    "$NuGetPackages" = (Get-ChildItem -Recurse $NuGetPackages);
     "$CoreXTPackages" = (Get-ChildItem "$CoreXTPackages\Microsoft.VisualStudio.Threading.VSInsertionMetadata.$InsertionMetadataVersion.nupkg");
 }

azure-pipelines/artifacts/_all.ps1

@@ -1,18 +1,24 @@
 #!/usr/bin/env pwsh
 
-# This script returns all the artifacts that should be collected after a build.
-#
-# Each powershell artifact is expressed as an object with these properties:
-#  Source          - the full path to the source file
-#  ArtifactName    - the name of the artifact to upload to
-#  ContainerFolder - the relative path within the artifact in which the file should appear
-#
-# Each artifact aggregating .ps1 script should return a hashtable:
-#   Key = path to the directory from which relative paths within the artifact should be calculated
-#   Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact.
-#           FileInfo objects are also allowed.
-
-$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+<#
+.SYNOPSIS
+    This script returns all the artifacts that should be collected after a build.
+    Each powershell artifact is expressed as an object with these properties:
+      Source          - the full path to the source file
+      ArtifactName    - the name of the artifact to upload to
+      ContainerFolder - the relative path within the artifact in which the file should appear
+    Each artifact aggregating .ps1 script should return a hashtable:
+      Key = path to the directory from which relative paths within the artifact should be calculated
+      Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact.
+              FileInfo objects are also allowed.
+.PARAMETER Force
+    Executes artifact scripts even if they have already been uploaded.
+#>
+
+param (
+    [string]$ArtifactNameSuffix,
+    [switch]$Force
+)
 
 Function EnsureTrailingSlash($path) {
     if ($path.length -gt 0 -and !$path.EndsWith('\') -and !$path.EndsWith('/')) {
@@ -22,35 +28,43 @@ Function EnsureTrailingSlash($path) {
     $path.Replace('\', [IO.Path]::DirectorySeparatorChar)
 }
 
-Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse |% {
-    $ArtifactName = $_.BaseName
+Function Test-ArtifactUploaded($artifactName) {
+    $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())"
+    Test-Path "env:$varName"
+}
 
-    $totalFileCount = 0
-    $fileGroups = & $_
-    if ($fileGroups) {
-        $fileGroups.GetEnumerator() | % {
-            $BaseDirectory = New-Object Uri ((EnsureTrailingSlash $_.Key.ToString()), [UriKind]::Absolute)
-            $_.Value | % {
-                if ($_.GetType() -eq [IO.FileInfo] -or $_.GetType() -eq [IO.DirectoryInfo]) {
-                    $_ = $_.FullName
-                }
+Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % {
+    $ArtifactName = $_.BaseName
+    if ($Force -or !(Test-ArtifactUploaded($ArtifactName + $ArtifactNameSuffix))) {
+        $totalFileCount = 0
+        $fileGroups = & $_
+        if ($fileGroups) {
+            $fileGroups.GetEnumerator() | % {
+                $BaseDirectory = New-Object Uri ((EnsureTrailingSlash $_.Key.ToString()), [UriKind]::Absolute)
+                $_.Value | ? { $_ } | % {
+                    if ($_.GetType() -eq [IO.FileInfo] -or $_.GetType() -eq [IO.DirectoryInfo]) {
+                        $_ = $_.FullName
+                    }
 
-                $artifact = New-Object -TypeName PSObject
-                Add-Member -InputObject $artifact -MemberType NoteProperty -Name ArtifactName -Value $ArtifactName
+                    $artifact = New-Object -TypeName PSObject
+                    Add-Member -InputObject $artifact -MemberType NoteProperty -Name ArtifactName -Value $ArtifactName
 
-                $SourceFullPath = New-Object Uri ($BaseDirectory, $_)
-                Add-Member -InputObject $artifact -MemberType NoteProperty -Name Source -Value $SourceFullPath.LocalPath
+                    $SourceFullPath = New-Object Uri ($BaseDirectory, $_)
+                    Add-Member -InputObject $artifact -MemberType NoteProperty -Name Source -Value $SourceFullPath.LocalPath
 
-                $RelativePath = [Uri]::UnescapeDataString($BaseDirectory.MakeRelative($SourceFullPath))
-                Add-Member -InputObject $artifact -MemberType NoteProperty -Name ContainerFolder -Value (Split-Path $RelativePath)
+                    $RelativePath = [Uri]::UnescapeDataString($BaseDirectory.MakeRelative($SourceFullPath))
+                    Add-Member -InputObject $artifact -MemberType NoteProperty -Name ContainerFolder -Value (Split-Path $RelativePath)
 
-                Write-Output $artifact
-                $totalFileCount += 1
+                    Write-Output $artifact
+                    $totalFileCount += 1
+                }
             }
         }
-    }
 
-    if ($totalFileCount -eq 0) {
-        Write-Warning "No files found for the `"$ArtifactName`" artifact."
+        if ($totalFileCount -eq 0) {
+            Write-Warning "No files found for the `"$ArtifactName`" artifact."
+        }
+    } else {
+        Write-Host "Skipping $ArtifactName because it has already been uploaded." -ForegroundColor DarkGray
     }
 }

azure-pipelines/artifacts/_pipelines.ps1

@@ -7,4 +7,9 @@ param (
 
 & "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% {
     Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)"
+
+    # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts
+    # will skip this one from a check in the _all.ps1 script.
+    $varName = "ARTIFACTUPLOADED_$($_.Name.ToUpper())"
+    Write-Host "##vso[task.setvariable variable=$varName]true"
 }

azure-pipelines/artifacts/_stage_all.ps1

@@ -38,7 +38,7 @@ function Create-SymbolicLink {
 }
 
 # Stage all artifacts
-$Artifacts = & "$PSScriptRoot\_all.ps1"
+$Artifacts = & "$PSScriptRoot\_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix
 $Artifacts |% {
     $DestinationFolder = (Join-Path (Join-Path $ArtifactStagingFolder "$($_.ArtifactName)$ArtifactNameSuffix") $_.ContainerFolder).TrimEnd('\')
     $Name = "$(Split-Path $_.Source -Leaf)"

azure-pipelines/build.yml

@@ -39,6 +39,11 @@ jobs:
     - template: microbuild.after.yml
       parameters:
         EnableAPIScan: ${{ parameters.EnableAPIScan }}
+    # Repeat this step to scoop up any artifacts that would only be collected after running microbuild.after.yml
+    - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)"
+      failOnStderr: true
+      displayName: Publish artifacts
+      condition: succeededOrFailed()
 
 - job: Linux
   condition: ne(variables['OptProf'], 'true')

azure-pipelines/microbuild.after.yml

@@ -32,6 +32,15 @@ steps:
   displayName: Publish InsertionOutputs as Azure DevOps artifacts
   condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
 
+- task: ManifestGeneratorTask@0
+  displayName: Software Bill of Materials generation
+  inputs:
+    BuildDropPath: $(System.DefaultWorkingDirectory)/bin/Microsoft.VisualStudio.Threading/$(BuildConfiguration)
+    BuildComponentPath: $(System.DefaultWorkingDirectory)/obj/src/Microsoft.VisualStudio.Threading
+
+- powershell: Copy-Item -Recurse "$(System.DefaultWorkingDirectory)/bin/Microsoft.VisualStudio.Threading/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet"
+  displayName: Publish Software Bill of Materials
+
 - task: Ref12Analyze@0
   displayName: Ref12 (Codex) Analyze
   inputs:

azure-pipelines/official.yml

@@ -53,6 +53,7 @@ stages:
     push_to_ci: true
     NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages
     SignTypeSelection: ${{ parameters.SignTypeSelection }}
+    Packaging.EnableSBOMSigning: true
 
   jobs:
   - template: build.yml