[v3.x] Add 1ES CI pipelines

NuGet.config

@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
  <packageSources>
- <add key="nuget.org" value="https://www.nuget.org/api/v2/" />
+ <clear />
+ <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
  <add key="AzureFunctionsTempStaging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
  <add key="azure-appservice-test" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/azure-appservice-test%40Local/nuget/v3/index.json" />
  <add key="Microsoft.Azure.Functions.PowerShellWorker" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/Microsoft.Azure.Functions.PowerShellWorker/nuget/v3/index.json" />
@@ -10,4 +11,4 @@
  <!-- dotnet-tools is required for Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit -->
  <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"/>
  </packageSources>
-</configuration>
+</configuration>

azure-pipelines.yml

@@ -49,9 +49,6 @@ jobs:
  buildNumberSuffix: $[variables.buildNumberSuffixTemp]
  packSuffixSwitch: $[variables.packSuffixSwitchTemp]
  emgSuffixSwitch: $[variables.emgSuffixSwitchTemp]
- majorVersion: $[dependencies.InitializePipeline.outputs['Initialize.MajorVersion']]
- minorVersion: $[dependencies.InitializePipeline.outputs['Initialize.MinorVersion']]
- patchVersion: $[dependencies.InitializePipeline.outputs['Initialize.PatchVersion']]
  pool:
  name: '1ES-Hosted-AzFunc'
  demands:
@@ -68,7 +65,7 @@ jobs:
  displayName: "Build artifacts"
  inputs:
  filePath: '$(Build.Repository.LocalPath)\build\build-extensions.ps1'
- arguments: '-buildNumber "$(buildNumberSuffix)" -majorMinorVersion "$(majorVersion).$(minorVersion)" -patchVersion "$(patchVersion)" -suffix "$(suffix)" -commitHash "$(Build.SourceVersion)"'
+ arguments: '-buildNumber "$(buildNumberSuffix)" -suffix "$(suffix)" -commitHash "$(Build.SourceVersion)"'
  - task: PowerShell@2
  condition: eq(variables['RUNBUILDFORINTEGRATIONTESTS'], 'True')
  displayName: "Update host references"
@@ -112,14 +109,6 @@ jobs:
  publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
  allowPackageConflicts: true
 
- - task: DotNetCoreCLI@2
- displayName: 'Build performance package'
- inputs:
- command: 'custom'
- custom: 'pack'
- arguments: '-o WebJobs.Script.Performance.App'
- projects: |
- **\WebJobs.Script.Performance.App.csproj
  - task: DotNetCoreCLI@2
  displayName: 'Build Abstractions and ExtensionsMetadataGenerator'
  inputs:
@@ -325,13 +314,6 @@ jobs:
  Verbosity: 'Information'
  - publish: $(Build.Repository.LocalPath)\packages
  artifact: NugetPackages
- - task: ManifestGeneratorTask@0
- displayName: 'SBOM Generation Task - Performance'
- inputs:
- BuildDropPath: '$(Build.Repository.LocalPath)\WebJobs.Script.Performance.App'
- Verbosity: 'Information'
- - publish: $(Build.Repository.LocalPath)\WebJobs.Script.Performance.App
- artifact: Performance
 
 - job: RunUnitTests
  pool:

build/Get-AzureFunctionsVersion.ps1

@@ -0,0 +1,16 @@
+# Get major, minor and patchVersions
+$version = @{}
+[xml]$XMLContents = [xml](Get-Content -Path "$PSScriptRoot\common.props")
+$XMLContents.GetElementsByTagName("MajorVersion") | ForEach-Object {
+ $version.Major = $_.InnerText
+}
+
+$XMLContents.GetElementsByTagName("MinorVersion") | ForEach-Object {
+ $version.Minor = $_.InnerText
+}
+
+$XMLContents.GetElementsByTagName("PatchVersion") | ForEach-Object {
+ $version.Patch = $_.InnerText
+}
+
+return $version

build/build-extensions.ps1

@@ -1,12 +1,14 @@
 param (
  [string]$buildNumber,
- [string]$majorMinorVersion,
- [string]$patchVersion,
  [string]$suffix = "",
  [string]$commitHash = "N/A",
  [string]$hashesForHardlinksFile = "hashesForHardlinks.txt"
 )
 
+$version = & $PSScriptRoot\Get-AzureFunctionsVersion.ps1
+$majorMinorVersion = "$($version.Major).$($version.Minor)"
+$patchVersion = $version.Patch
+
 $extensionVersion = "$majorMinorVersion.$patchVersion"
 Write-Host "ExtensionVersion is $extensionVersion"
 Write-Host "BuildNumber is $buildNumber"

build/common.props

@@ -16,14 +16,22 @@
  <CodeAnalysisRuleSet>$(MSBuildThisFileDirectory)..\src.ruleset</CodeAnalysisRuleSet>
  <NoWarn>$(NoWarn);NU1701</NoWarn>
  <DebugType>embedded</DebugType>
- <UseSourceLink Condition="$(UseSourceLink) == '' And $(CI) != ''">true</UseSourceLink>
- <Deterministic>True</Deterministic>
+ <Deterministic>True</Deterministic>
+ <NugetAudit>false</NugetAudit>
  </PropertyGroup>
 
  <PropertyGroup Condition="'$(UseSourceLink)' == 'true'">
  <SourceLink>$(BaseIntermediateOutputPath)source_link.json</SourceLink>
  </PropertyGroup>
-
+
+ <PropertyGroup>
+ <!--
+ NOTE: source link does not actually work for v3.x in 1ES as the git remote url is Azure DevOps, not GitHub.
+ However, we do not release public nuget packages from this branch anymore so it is not even used.
+ -->
+ <UseSourceLink Condition="$(UseSourceLink) == '' And $(CI) != ''">true</UseSourceLink>
+ </PropertyGroup>
+
  <Target Name="GenerateSourceLink" BeforeTargets="CoreCompile" Condition="'$(UseSourceLink)' == 'true'">
  <PropertyGroup>
  <SrcRootDirectory>$([System.IO.Directory]::GetParent($(MSBuildThisFileDirectory.TrimEnd("\"))))</SrcRootDirectory>

build/initialize-pipeline.ps1

@@ -17,29 +17,21 @@ if ($buildReason -eq "PullRequest") {
 }
 
 # Get major, minor and patchVersions
-[xml]$XMLContents = [xml](Get-Content -Path ".\build\common.props")
-$XMLContents.GetElementsByTagName("MajorVersion") | ForEach-Object {
- $majorVersion = $_.InnerText
- Write-Host "##vso[task.setvariable variable=MajorVersion;isOutput=true]$majorVersion"
- Write-Host "Setting 'MajorVersion' to $majorVersion"
-}
+$version = & $PSScriptRoot\Get-AzureFunctionsVersion.ps1
+Write-Host "##vso[task.setvariable variable=MajorVersion;isOutput=true]$($version.Major)"
+Write-Host "Setting 'MajorVersion' to $($version.Major)"
 
-$XMLContents.GetElementsByTagName("MinorVersion") | ForEach-Object {
- $minorVersion = $_.InnerText
- Write-Host "##vso[task.setvariable variable=MinorVersion;isOutput=true]$minorVersion"
- Write-Host "Setting 'MinorVersion' to $minorVersion"
-}
+Write-Host "##vso[task.setvariable variable=MinorVersion;isOutput=true]$($version.Minor)"
+Write-Host "Setting 'MinorVersion' to $($version.Minor)"
 
-$XMLContents.GetElementsByTagName("PatchVersion") | ForEach-Object {
- $patchVersion = $_.InnerText
- Write-Host "##vso[task.setvariable variable=PatchVersion;isOutput=true]$patchVersion"
- Write-Host "Setting 'PatchVersion' to $patchVersion"
-}
+Write-Host "##vso[task.setvariable variable=PatchVersion;isOutput=true]$($version.Patch)"
+Write-Host "Setting 'PatchVersion' to $($version.Patch)"
 
 #Update buildnumber with the same (Will be used by release pipelines)
-$customBuildNumber = "$majorVersion.$minorVersion.$patchVersion"
+$customBuildNumber = "$($version.Major).$($version.Minor).$($version.Patch)"
 if(($buildReason -eq "PullRequest") -or !($sourceBranch.ToLower().Contains("release")))
 {
  $customBuildNumber = "$customBuildNumber-$buildNumber"
 }
-Write-Host "##vso[build.updatebuildnumber]$customBuildNumber"
+
+Write-Host "##vso[build.updatebuildnumber]$customBuildNumber"

eng/ci/code-mirror.yml

@@ -0,0 +1,18 @@
+trigger:
+ branches:
+ include:
+ - v3.x
+ - release/3.*
+
+resources:
+ repositories:
+ - repository: eng
+ type: git
+ name: engineering
+ ref: refs/tags/release
+
+variables:
+ - template: ci/variables/cfs.yml@eng
+
+extends:
+ template: ci/code-mirror.yml@eng

eng/ci/integration-tests.yml

@@ -0,0 +1,37 @@
+trigger: none # ensure this is not ran as a CI build
+
+pr:
+ branches:
+ include:
+ - v3.x
+ - release/3.*
+
+resources:
+ repositories:
+ - repository: 1es
+ type: git
+ name: 1ESPipelineTemplates/1ESPipelineTemplates
+ ref: refs/tags/release
+ - repository: eng
+ type: git
+ name: engineering
+ ref: refs/tags/release
+
+variables:
+ - template: /eng/ci/templates/variables/build.yml@self
+ - template: /ci/variables/cfs.yml@eng
+
+extends:
+ template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+ parameters:
+ pool:
+ name: 1es-pool-azfunc
+ image: 1es-windows-2022
+ os: windows
+
+ stages:
+ - stage: Test
+ jobs:
+ - template: /eng/ci/templates/jobs/initialize-pipeline.yml@self
+ - template: /eng/ci/templates/official/jobs/run-non-e2e-tests.yml@self
+ - template: /eng/ci/templates/official/jobs/run-integration-tests.yml@self

eng/ci/official-build.yml

@@ -0,0 +1,54 @@
+trigger:
+ batch: true
+ branches:
+ include:
+ - v3.x
+ - release/3.*
+
+# CI only, does not trigger on PRs.
+pr: none
+
+resources:
+ repositories:
+ - repository: 1es
+ type: git
+ name: 1ESPipelineTemplates/1ESPipelineTemplates
+ ref: refs/tags/release
+ - repository: eng
+ type: git
+ name: engineering
+ ref: refs/tags/release
+
+variables:
+ - template: /eng/ci/templates/variables/build.yml@self
+ - template: /ci/variables/cfs.yml@eng
+ - name: buildNumber
+ value: $[ counter('build', 23000) ] # 23000 selected to be ahead of current host build
+
+extends:
+ template: v1/1ES.Official.PipelineTemplate.yml@1es
+ parameters:
+ pool:
+ name: 1es-pool-azfunc
+ image: 1es-windows-2022
+ os: windows
+
+ stages:
+ - stage: Initialize
+
+ jobs:
+ - template: /eng/ci/templates/jobs/initialize-pipeline.yml@self
+
+ - stage: Build
+ dependsOn: Initialize
+
+ jobs:
+ - template: /eng/ci/templates/official/jobs/build-artifacts-windows.yml@self
+
+ - stage: Test
+ dependsOn: Initialize
+
+ jobs:
+ - template: /eng/ci/templates/jobs/run-unit-tests.yml@self
+ - template: /eng/ci/templates/official/jobs/run-non-e2e-tests.yml@self
+ - template: /eng/ci/templates/official/jobs/run-integration-tests.yml@self

eng/ci/public-build.yml

@@ -0,0 +1,38 @@
+# This build is used for public PR and CI builds.
+
+trigger:
+ batch: true
+ branches:
+ include:
+ - v3.x
+ - release/3.*
+
+pr:
+ branches:
+ include:
+ - v3.x
+ - release/3.*
+
+resources:
+ repositories:
+ - repository: 1es
+ type: git
+ name: 1ESPipelineTemplates/1ESPipelineTemplates
+ ref: refs/tags/release
+
+variables:
+ - template: /eng/ci/templates/variables/build.yml@self
+
+extends:
+ template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+ parameters:
+ pool:
+ name: 1es-pool-azfunc-public
+ image: 1es-windows-2022
+ os: windows
+
+ stages:
+ - stage: Test
+ jobs:
+ - template: /eng/ci/templates/jobs/initialize-pipeline.yml@self
+ - template: /eng/ci/templates/jobs/run-unit-tests.yml@self

eng/ci/templates/install-dotnet.yml

@@ -0,0 +1,19 @@
+steps:
+
+- task: UseDotNet@2
+ displayName: Install .NET Core 2.1 runtime (tests)
+ inputs:
+ packageType: runtime
+ version: 2.1.x
+
+- task: UseDotNet@2
+ displayName: Install .NET Core 3.1
+ inputs:
+ packageType: 'sdk'
+ version: '3.1.x'
+
+- task: UseDotNet@2
+ displayName: Install .NET 8
+ inputs:
+ packageType: sdk
+ version: 8.0.x

eng/ci/templates/jobs/initialize-pipeline.yml

@@ -0,0 +1,13 @@
+jobs:
+ - job: InitializePipeline
+ displayName: Initialize Pipeline
+
+ steps:
+ - task: PowerShell@2
+ displayName: 'Initialize'
+ name: Initialize
+ inputs:
+ filePath: build/initialize-pipeline.ps1
+ arguments: -buildNumber $(buildNumber)
+ showWarnings: true
+ pwsh: true