Merge main to runtime async branch

.github/workflows/main-merge.yml

@@ -5,6 +5,9 @@ on:
   schedule:
     # Create a merge every 3 hours (works only for merges from `main`, others would need a `push` trigger).
     - cron: '0 */3 * * *'
+  push:
+    branches:
+      - main-vs-deps
   workflow_dispatch:
     inputs:
       configuration_file_branch:
@@ -17,15 +20,9 @@ permissions:
   pull-requests: write
 
 jobs:
-  # The config does not support multiple flows from the same source branch,
-  # so we need to run separately for each duplicate source branch (https://github.com/dotnet/arcade/issues/15586).
   merge:
+    if: github.repository == 'dotnet/roslyn'
     uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main
     with:
       configuration_file_path: 'eng/config/branch-merge.jsonc'
       configuration_file_branch: ${{ inputs.configuration_file_branch || 'main' }}
-  merge-2:
-    uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main
-    with:
-      configuration_file_path: 'eng/config/branch-merge-2.jsonc'
-      configuration_file_branch: ${{ inputs.configuration_file_branch || 'main' }}

azure-pipelines-official.yml

@@ -7,6 +7,7 @@ trigger:
     - release/dev17.*
     - release/dev18.*
     - features/lsp_tools_host
+    - features/runtime-async
     exclude:
     - release/dev17.0
 pr: none
@@ -163,7 +164,7 @@ extends:
 
           - output: pipelineArtifact
             displayName: 'Publish Ngen Logs'
-            condition: succeeded()
+            condition: and(succeeded(), ${{ not(parameters.SkipApplyOptimizationData) }})
             targetPath: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)\ngen'
             artifactName: 'NGen Logs'
             publishLocation: Container
@@ -304,7 +305,7 @@ extends:
                        -configuration $(BuildConfiguration)
                        -officialBuildId $(Build.BuildNumber)
                        -officialSkipTests $(SkipTests)
-                       -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
+                       -officialSkipApplyOptimizationData ${{ parameters.SkipApplyOptimizationData }}
                        -officialSourceBranchName $(SourceBranchName)
                        -officialIbcDrop $(IbcDrop)
                        -officialVisualStudioDropAccessToken $(_DevDivDropAccessToken)

azure-pipelines-pr-validation.yml

@@ -98,7 +98,7 @@ extends:
 
           - output: pipelineArtifact
             displayName: 'Publish Ngen Logs'
-            condition: succeeded()
+            condition: and(succeeded(), ${{ not(parameters.SkipApplyOptimizationData) }})
             targetPath: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)\ngen'
             artifactName: 'NGen Logs'
             publishLocation: Container
@@ -242,7 +242,7 @@ extends:
                        -configuration $(BuildConfiguration)
                        -officialBuildId $(OriginalBuildNumber)
                        -officialSkipTests $(SkipTests)
-                       -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
+                       -officialSkipApplyOptimizationData ${{ parameters.SkipApplyOptimizationData }}
                        -officialSourceBranchName $(SourceBranchName)
                        -officialIbcDrop $(IbcDrop)
                        -officialVisualStudioDropAccessToken $(_DevDivDropAccessToken)

azure-pipelines.yml

@@ -406,7 +406,7 @@ stages:
       - powershell: eng/build.ps1 -configuration Release -prepareMachine -ci -restore -binaryLogName Restore.binlog 
         displayName: Restore
 
-      - powershell: eng/build.ps1 -configuration Release -prepareMachine -ci -build -pack -publish -sign -binaryLogName Build.binlog /p:DotnetPublishUsingPipelines=true
+      - powershell: eng/build.ps1 -configuration Release -prepareMachine -ci -build -pack -publish -sign -binaryLogName Build.binlog /p:DotnetPublishUsingPipelines=true /p:ContinuousIntegrationBuildCorrectness=true
         displayName: Build
 
       # While this task is not executed in the official build, this serves as a PR check for whether symbol exclusions

docs/compilers/CSharp/Compiler Breaking Changes - DotNet 9.md

@@ -183,4 +183,4 @@ of the value is as follows:
 5. `omit_if_default`.  Redundant modifiers are disallowed.  For example a private class member will be disallowed from
    using `private`, and a public interface member will be disallowed from using `public`.  This is the option to use
    if you feel that restating the accessibility when it matches what the language chooses by default is redundant and
-   should be disalloed.
+   should be disallowed.

docs/features/field-nullability.md

@@ -0,0 +1,23 @@
+# `field` keyword nullable analysis implementation
+
+See also [specification](https://github.com/dotnet/csharplang/blob/94205582d0f5c73e5765cb5888311c2f14890b95/proposals/field-keyword.md#nullability).
+
+## Symbol API behavior
+
+In order to decide the NullableAnnotation of a SynthesizedBackingFieldSymbol we need to:
+1. Decide if the getter associated with this field requires a null-resilience analysis.
+2. If it *doesn't* require such an analysis, then the nullable annotation of the property is used.
+3. If it *does*, then a binding+nullable analysis of the getter must be performed to decide the nullability of the backing field.
+
+There are some significant problems with directly exposing the nullable annotation on this field symbol.
+1. Cost. It's not clear if it's OK for reading the NullableAnnotation off a field to cause us to bind and nullable analyze something. If some tooling is traversing member symbols for indexing or some such, it may be a problem if that causes unexpected method binding to occur where it didn't before.
+2. Stack cycles. In the process of binding+flow analyzing, we may want to access the nullable annotation of the field. If we are *already* in the process of determining that field's nullable annotation, then we would need to "short-circuit" and return some "placeholder" value. We would need to take care to isolate the "internal" implementation from any public implementation, in order to ensure that a consistent answer is given to users of the public API.
+...
+
+It's tempting to avoid exposing the inferred nullable annotation in the public symbol model. This might be problematic for automated tooling which is trying to reason about nullable initialization. For example, if a diagnostic suppressor wants to suppress `CS8618` nullable initialization warnings under certain conditions. How should it decide whether/why a property like `string Prop => field ??= GetValue();` requires initialization in constructors? It's unclear to me whether this matters.
+
+In the interests of *simplicity* (first make it correct, then make it fast), I'd like to move forward by *not* exposing the inferred annotation in the symbol model. Rather, we introduce a new internal API `SynthesizedBackingFieldSymbol.GetInferredNullableAnnotation()`, which `NullableWalker` will use to decide initial nullable state, report warnings, and so on. The implementation simply does an on-demand binding of the get accessor, then nullable analyzes and decides the nullable annotation. Care needs to be taken to avoid using this API in a re-entrant manner--so, the `NullableWalker` passes which are used to infer the nullability annotation must avoid calling it.
+
+This means that the ordinary `FieldSymbol.TypeWithAnnotations` API would not expose the inferred nullability, and neither would `IFieldSymbol.Type` or `IFieldSymbol.NullableAnnotation`. Instead the field's nullability would always match the property's. This is something I would actually like to fix, maybe before merging to main. It feels like Quick Info, etc., should expose the inferred nullability through the ordinary APIs.
+
+Once we get a handle on the behaviors of the getter null resilience, we can start talking about how to reduce cost associated with it. For example, we could try to reduce redundant binding, by forcing nullable annotations to be inferred prior to analyzing certain methods. Before processing a constructor in `MethodCompiler`, for example, maybe we would identify the getters that need to be compiled for purposes of inference, and force those to compile first.

eng/Version.Details.xml

@@ -8,9 +8,9 @@
       <SourceBuild RepoName="source-build-externals" ManagedOnly="true" />
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="10.0.616001">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="10.0.616902">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
-      <Sha>8f9f6fa14523b2f2475ea0a86be1cf60b5de5336</Sha>
+      <Sha>b99d40df247e865fb0a9de15bdcfdfa7f550a55e</Sha>
       <SourceBuild RepoName="source-build-reference-packages" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="System.CommandLine" Version="2.0.0-beta4.24528.1">
@@ -122,19 +122,19 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.25111.5">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.25164.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5da211e1c42254cb35e7ef3d5a8428fb24853169</Sha>
+      <Sha>5ba9ca776c1d0bb72b2791591e54cf51fc52dfee</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.25111.5">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.25164.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5da211e1c42254cb35e7ef3d5a8428fb24853169</Sha>
+      <Sha>5ba9ca776c1d0bb72b2791591e54cf51fc52dfee</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.25111.5">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.25164.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5da211e1c42254cb35e7ef3d5a8428fb24853169</Sha>
+      <Sha>5ba9ca776c1d0bb72b2791591e54cf51fc52dfee</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DiaSymReader" Version="2.0.0">
       <Uri>https://github.com/dotnet/symreader</Uri>
@@ -150,9 +150,9 @@
       <Uri>https://github.com/dotnet/roslyn</Uri>
       <Sha>5d10d428050c0d6afef30a072c4ae68776621877</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.25111.5">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.25164.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5da211e1c42254cb35e7ef3d5a8428fb24853169</Sha>
+      <Sha>5ba9ca776c1d0bb72b2791591e54cf51fc52dfee</Sha>
     </Dependency>
     <Dependency Name="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0-preview.23468.1">
       <Uri>https://github.com/dotnet/roslyn-analyzers</Uri>

eng/common/core-templates/steps/generate-sbom.yml

@@ -38,7 +38,7 @@ steps:
       PackageName: ${{ parameters.packageName }}
       BuildDropPath: ${{ parameters.buildDropPath }}
       PackageVersion: ${{ parameters.packageVersion }}
-      ManifestDirPath: ${{ parameters.manifestDirPath }}
+      ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME)
       ${{ if ne(parameters.IgnoreDirectories, '') }}:
         AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
 

eng/common/generate-sbom-prep.ps1

@@ -4,18 +4,26 @@ Param(
 
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
 Write-Host "Creating dir $ManifestDirPath"
+
 # create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
+if (!(Test-Path -path $SbomGenerationDir))
 {
-  New-Item -ItemType Directory -path $ManifestDirPath
-  Write-Host "Successfully created directory $ManifestDirPath"
+  New-Item -ItemType Directory -path $SbomGenerationDir
+  Write-Host "Successfully created directory $SbomGenerationDir"
 }
 else{
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 }
 
 Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"

eng/common/generate-sbom-prep.sh

@@ -14,19 +14,24 @@ done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 . $scriptroot/pipeline-logging-functions.sh
 
+
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
 manifest_dir=$1
 
-if [ ! -d "$manifest_dir" ] ; then
-  mkdir -p "$manifest_dir"
-  echo "Sbom directory created." $manifest_dir
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+  mkdir -p "$sbom_generation_dir"
+  echo "Sbom directory created." $sbom_generation_dir
 else
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 fi
 
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
 echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
 echo "Artifact name after : "$safe_artifact_name
 export ARTIFACT_NAME=$safe_artifact_name
 echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"

eng/common/templates-official/job/job.yml

@@ -16,6 +16,7 @@ jobs:
         parameters:
           PackageVersion: ${{ parameters.packageVersion }}
           BuildDropPath: ${{ parameters.buildDropPath }}
+          ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
           publishArtifacts: false
 
     # publish artifacts

eng/common/tools.ps1

@@ -42,7 +42,7 @@
 [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
 
 # Enable repos to use a particular version of the on-line dotnet-install scripts.
-#    default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
+#    default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1
 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
 
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -262,7 +262,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
     $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
-    $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
+    $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
 
     Retry({
       Write-Host "GET $uri"

eng/common/tools.sh

@@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true}
 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
 
 # Enable repos to use a particular version of the on-line dotnet-install scripts.
-#    default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
+#    default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh
 dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
 
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -295,7 +295,7 @@ function with_retries {
 function GetDotNetInstallScript {
   local root=$1
   local install_script="$root/dotnet-install.sh"
-  local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+  local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
 
   if [[ ! -a "$install_script" ]]; then
     mkdir -p "$root"

eng/config/PublishData.json

@@ -175,7 +175,7 @@
       "vsBranch": "feature/d18initial",
       "vsMajorVersion": 18,
       "insertionTitlePrefix": "[d18.0 P1]",
-      "insertionCreateDraftPR": true
+      "insertionCreateDraftPR": false
     },
     "main": {
       "nugetKind": [

eng/config/branch-merge.jsonc

@@ -1,8 +1,13 @@
 // Used by .github/workflows/main-merge.yml
 {
     "merge-flow-configurations": {
-        // Merge any main changes to release/dev18.0.
+        // Merge any main changes to main-vs-deps.
         "main": {
+            "MergeToBranch": "main-vs-deps",
+            "ExtraSwitches": "-QuietComments"
+        },
+        // Merge any main-vs-deps changes to release/dev18.0.
+        "main-vs-deps": {
             "MergeToBranch": "release/dev18.0",
             "ExtraSwitches": "-QuietComments"
         }

global.json

@@ -1,18 +1,18 @@
 {
   "sdk": {
-    "version": "9.0.103",
+    "version": "9.0.104",
     "allowPrerelease": false,
     "rollForward": "patch"
   },
   "tools": {
-    "dotnet": "9.0.103",
+    "dotnet": "9.0.104",
     "vs": {
       "version": "17.8.0"
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25111.5",
-    "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25111.5",
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25164.2",
+    "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25164.2",
     "Microsoft.Build.Traversal": "3.4.0"
   }
 }

src/Analyzers/CSharp/Analyzers/CSharpAnalyzersResources.resx

@@ -431,4 +431,7 @@
   <data name="Use_unbound_generic_type" xml:space="preserve">
     <value>Use unbound generic type</value>
   </data>
+  <data name="Implement_with_Copilot" xml:space="preserve">
+    <value>Implement with Copilot</value>
+  </data>
 </root>