[7.0.4xx] Migrate to 1ES templates for internal builds

.vsts-pr.yml

@@ -26,7 +26,6 @@ variables:
   - group: DotNet-Installer-SDLValidation-Params
   - name: _PublishUsingPipelines
     value: true
-
 # Default to running tests in PRs and public CI, but not in official builds
 - name: _WindowsTestArg
   value: '-test'
@@ -37,10 +36,8 @@ variables:
     value: ''
   - name: _NonWindowsTestArg
     value: ''
-
 - name: _InternalRuntimeDownloadArgs
   value: ''
-
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNetBuilds storage account read tokens
   - name: _InternalRuntimeDownloadArgs
@@ -63,7 +60,7 @@ stages:
     - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration
       artifact: BuildConfiguration
       displayName: Publish Build Config
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Windows_NT
       pool:
@@ -107,7 +104,7 @@ stages:
               _TestArg: ''
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - template: /eng/build.yml
+    - template: /eng/build-pr.yml
       parameters:
         agentOs: Windows_NT
         pool:
@@ -139,7 +136,7 @@ stages:
               _TestArg: ''
         pgoInstrument: true
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Linux
       pool:
@@ -283,7 +280,7 @@ stages:
             _TestArg: $(_NonWindowsTestArg)
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - template: /eng/build.yml
+    - template: /eng/build-pr.yml
       parameters:
         agentOs: Linux
         pool:
@@ -309,7 +306,7 @@ stages:
               _TestArg: ''
         pgoInstrument: true
 
-  - template: /eng/build.yml
+  - template: /eng/build-pr.yml
     parameters:
       agentOs: Darwin
       pool:
@@ -332,26 +329,7 @@ stages:
 
   - template: /eng/common/templates/jobs/source-build.yml
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
-
-  # For .NET 7.0, source-build currently supports 7.0.1xx.  Disable tarball build validation in all other branches.
-  # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-  #   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
-  #     parameters:
-  #       dependsOn: Source_Build_Create_Tarball
-
-  # https://github.com/dotnet/core-sdk/issues/248
-  # - template: /eng/build.yml
-  #   parameters:
-  #     agentOs: FreeBSD
-  #     queue:
-  #       name: dnceng-freebsd-internal
-  #       timeoutInMinutes: 180
-  #       matrix:
-  #         Build_Release:
-  #           _BuildConfig: Release
-  #           _BuildArchitecture: 'x64'
-  #           _AdditionalBuildParameters: '/p:DisableSourceLink=true /p:DISABLE_CROSSGEN=true'
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/job/publish-build-assets.yml
@@ -370,7 +348,6 @@ stages:
             name: NetCore1ESPool-Svc-Internal
             demands: ImageOverride -equals windows.vs2017.amd64
 
-
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng\common\templates\post-build\post-build.yml
     parameters:

NuGet.config

@@ -10,19 +10,14 @@
     <add key="darc-pub-DotNet-msbuild-Trusted-3ebbd7c" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-DotNet-msbuild-Trusted-3ebbd7c4/nuget/v3/index.json" />
     <!--  End: Package sources from DotNet-msbuild-Trusted -->
     <!--  Begin: Package sources from dotnet-aspnetcore -->
-    <add key="darc-int-dotnet-aspnetcore-d0ca5a8-1" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-aspnetcore-d0ca5a8d-1/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-aspnetcore -->
     <!--  Begin: Package sources from dotnet-emsdk -->
-    <add key="darc-pub-dotnet-emsdk-ec6dc98" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-emsdk-ec6dc98e/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-emsdk -->
     <!--  Begin: Package sources from dotnet-runtime -->
-    <add key="darc-int-dotnet-runtime-dff486f" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-dff486f2/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-runtime -->
     <!--  Begin: Package sources from dotnet-sdk -->
-    <add key="darc-int-dotnet-sdk-2b14eab" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-sdk-2b14eab8/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-sdk -->
     <!--  Begin: Package sources from dotnet-windowsdesktop -->
-    <add key="darc-int-dotnet-windowsdesktop-8e24710" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-windowsdesktop-8e247102/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-windowsdesktop -->
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
@@ -44,15 +39,11 @@
   <disabledPackageSources>
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
     <!--  Begin: Package sources from dotnet-aspnetcore -->
-    <add key="darc-int-dotnet-aspnetcore-d0ca5a8-1" value="true" />
     <!--  End: Package sources from dotnet-aspnetcore -->
     <!--  Begin: Package sources from dotnet-runtime -->
-    <add key="darc-int-dotnet-runtime-dff486f" value="true" />
     <!--  Begin: Package sources from dotnet-sdk -->
-    <add key="darc-int-dotnet-sdk-2b14eab" value="true" />
     <!--  End: Package sources from dotnet-sdk -->
     <!--  Begin: Package sources from dotnet-windowsdesktop -->
-    <add key="darc-int-dotnet-windowsdesktop-8e24710" value="true" />
     <!--  End: Package sources from dotnet-windowsdesktop -->
     <!--  End: Package sources from dotnet-runtime -->
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->

eng/Version.Details.xml

@@ -192,18 +192,18 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.24163.12">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.24172.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bbc75d8a5897763eaf6945c263759930e02dd32b</Sha>
+      <Sha>a14feb9f4edafbe99fc870b306615b639af6ae43</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.24163.12">
+    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.24172.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bbc75d8a5897763eaf6945c263759930e02dd32b</Sha>
+      <Sha>a14feb9f4edafbe99fc870b306615b639af6ae43</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.24163.12">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.24172.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bbc75d8a5897763eaf6945c263759930e02dd32b</Sha>
+      <Sha>a14feb9f4edafbe99fc870b306615b639af6ae43</Sha>
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="7.0.0-alpha.1.24121.2">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>

eng/Versions.props

@@ -33,7 +33,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.24163.12</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.24172.6</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/build-pr.yml

@@ -0,0 +1,156 @@
+parameters:
+  # Agent OS identifier and used as job name
+  agentOs: ''
+
+  # Agent pool
+  pool: {}
+
+  # Additional variables
+  variables: {}
+
+  # Build strategy - matrix
+  strategy: {}
+
+  # Job timeout
+  timeoutInMinutes: 180
+
+  # Publish using pipelines
+  enablePublishUsingPipelines: true
+
+phases:
+- template: /eng/common/templates/job/job.yml
+  parameters:
+    ${{ if parameters.pgoInstrument }}:
+      name: PGO_${{ parameters.agentOs }}
+    ${{ if not(parameters.pgoInstrument) }}:
+      name: ${{ parameters.agentOs }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      enableMicrobuild: true
+    enablePublishBuildAssets: true
+    enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
+    enableTelemetry: true
+    helixRepo: dotnet/installer
+    pool: ${{ parameters.pool }}
+    ${{ if ne(parameters.strategy, '') }}:
+      strategy: ${{ parameters.strategy }}
+    workspace:
+      clean: all
+
+    variables:
+      - _PgoInstrument: ''
+      - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+        - _PackArg: '-pack'
+      - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+        - _PackArg: '--pack'
+      - ${{ if parameters.pgoInstrument }}:
+        - _PgoInstrument: '/p:PgoInstrument=true'
+        - _PackArg: ''
+      - _AgentOSName: ${{ parameters.agentOs }}
+      - _TeamName: Roslyn-Project-System
+      - _SignType: test
+      - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        - group: DotNet-Symbol-Server-PATs
+        - group: DotNet-HelixApi-Access
+        - group: DotNet-Blob-Feed
+        - _DotNetPublishToBlobFeed: true
+        - _PushToVSFeed: true
+        - _SignType: real
+        - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+                  /p:DotNetSignType=$(_SignType)
+                  /p:TeamName=$(_TeamName)
+                  /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+                  $(_PgoInstrument)
+
+    steps:
+    - checkout: self
+      clean: true
+    - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - script: build.cmd
+                  $(_TestArg) $(_PackArg)
+                  -publish -ci -sign
+                  -Configuration $(_BuildConfig)
+                  -Architecture $(_BuildArchitecture)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+        env:
+          DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE)
+
+    - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+    - ${{ if eq(parameters.agentOs, 'Linux') }}:
+      - script: ./build.sh
+                  $(_TestArg) $(_PackArg)
+                  --publish --ci
+                  --noprettyprint
+                  --configuration $(_BuildConfig)
+                  $(_DockerParameter)
+                  --architecture $(_BuildArchitecture)
+                  $(_LinuxPortable)
+                  $(_RuntimeIdentifier)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+
+    - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
+      - script: ./build.sh
+                  $(_TestArg)
+                  --pack --publish --ci
+                  --noprettyprint
+                  --configuration $(_BuildConfig)
+                  --architecture $(_BuildArchitecture)
+                  $(_RuntimeIdentifier)
+                  $(_BuildArgs)
+                  $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
+        displayName: Build
+
+    - task: PublishTestResults@2
+      displayName: Publish Test Results
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
+        testRunTitle: '$(_AgentOSName)_$(Agent.JobName)'
+        platform: '$(BuildPlatform)'
+        configuration: '$(_BuildConfig)'
+      condition: ne(variables['_TestArg'], '')
+
+    - task: CopyFiles@2
+      displayName: Gather Logs
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)/artifacts'
+        Contents: |
+          log/$(_BuildConfig)/**/*
+          TestResults/$(_BuildConfig)/**/*
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+      continueOnError: true
+      condition: always()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Logs to VSTS
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)'
+        publishLocation: Container
+      continueOnError: true
+      condition: always()

eng/build.yml

@@ -18,7 +18,7 @@ parameters:
   enablePublishUsingPipelines: true
 
 phases:
-- template: /eng/common/templates/job/job.yml
+- template: /eng/common/templates-official/job/job.yml
   parameters:
     ${{ if parameters.pgoInstrument }}:
       name: PGO_${{ parameters.agentOs }}
@@ -28,9 +28,6 @@ phases:
     ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
       enableMicrobuild: true
     enablePublishBuildAssets: true
-    # Don't use built-in arcade logic for publishing artifacts and test results, as we manually do this and give better names
-    #enablePublishBuildArtifacts: true
-    #enablePublishTestResults: true
     enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
     enableTelemetry: true
     helixRepo: dotnet/installer
@@ -49,12 +46,10 @@ phases:
       - ${{ if parameters.pgoInstrument }}:
         - _PgoInstrument: '/p:PgoInstrument=true'
         - _PackArg: ''
-
       - _AgentOSName: ${{ parameters.agentOs }}
       - _TeamName: Roslyn-Project-System
       - _SignType: test
       - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
-
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
         - group: DotNet-HelixApi-Access
@@ -101,6 +96,7 @@ phases:
             arguments: $(Build.SourcesDirectory)/NuGet.config $Token
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
     - ${{ if eq(parameters.agentOs, 'Linux') }}:
       - script: ./build.sh
                   $(_TestArg) $(_PackArg)
@@ -144,13 +140,13 @@ phases:
       inputs:
         SourceFolder: '$(Build.SourcesDirectory)/artifacts'
         Contents: |
-         log/$(_BuildConfig)/**/*
-         TestResults/$(_BuildConfig)/**/*
+          log/$(_BuildConfig)/**/*
+          TestResults/$(_BuildConfig)/**/*
         TargetFolder: '$(Build.ArtifactStagingDirectory)'
       continueOnError: true
       condition: always()
 
-    - task: PublishBuildArtifacts@1
+    - task: 1ES.PublishBuildArtifacts@1
       displayName: Publish Logs to VSTS
       inputs:
         PathtoPublish: '$(Build.ArtifactStagingDirectory)'