Merge master to features/readonly-members

azure-pipelines-official.yml

@@ -6,6 +6,8 @@ variables:
   SignType: real
   SkipTests: false
   SkipApplyOptimizationData: false
+  IbcSourceBranchName: 'master-vs-deps'
+  IbcDropId: '75e3797e1105a4da4c10dddda76c3b9398f7725a/223453/935479/1'
 
 jobs:
 - job: OfficialBuild
@@ -19,8 +21,6 @@ jobs:
   timeoutInMinutes: 360
 
   variables:
-    VisualStudio.MajorVersion: 16
-    VisualStudio.ChannelName: 'int.d16.0stg'
     VisualStudio.DropName: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber)
 
   steps:
@@ -59,8 +59,10 @@ jobs:
               -officialBuildId $(Build.BuildNumber)
               -officialSkipTests $(SkipTests)
               -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
+              -officialSourceBranchName $(Build.SourceBranchName)
+              -officialIbcSourceBranchName $(IbcSourceBranchName)
+              -officialIbcDropId $(IbcDropId)
               /p:RepositoryName=$(Build.Repository.Name)
-              /p:VisualStudioIbcSourceBranchName="master-vs-deps"
               /p:VisualStudioDropAccessToken=$(System.AccessToken)
               /p:VisualStudioDropName=$(VisualStudio.DropName)
               /p:DotNetSignType=$(SignType)
@@ -70,8 +72,6 @@ jobs:
               /p:PublishToSymbolServer=true
               /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
               /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
-              /p:IbcOptimizationDataDir=$(Build.SourcesDirectory)\.o\\
-              /p:VisualStudioIbcDropId=75e3797e1105a4da4c10dddda76c3b9398f7725a/223453/935479/1
     displayName: Build
     condition: succeeded()
 

docs/compilers/CSharp/Compiler Breaking Changes - VS2019.md

@@ -81,3 +81,26 @@ Each entry should include a short description of the break, followed by either a
 
 10. Previously, reference assemblies were emitted including embedded resources. In Visual Studio 2019, embedded resources are no longer emitted into ref assemblies.
   See https://github.com/dotnet/roslyn/issues/31197
+
+11. Ref structs now support disposal via pattern. A ref struct enumerator with an accessible `void Dispose()` instance method will now have it invoked at the end of enumeration, regardless of whether the struct type implements IDisposable:
+``` c#
+public class C
+{
+    public ref struct RefEnumerator
+    {
+        public int Current => 0;
+        public bool MoveNext() => false;
+        public void Dispose() => Console.WriteLine("Called in C# 8.0 only");
+    }
+
+    public RefEnumerator GetEnumerator() => new RefEnumerator();
+
+    public static void Main()
+    {
+        foreach(var x in new C())
+        {
+        }
+        // RefEnumerator.Dispose() will be called here in C# 8.0
+    }
+}
+```

docs/features/nullable-reference-types.md

@@ -318,6 +318,27 @@ new C?(); // error
 new List<C?>(); // ok
 ```
 
+## Generated code
+Older code generation strategies may not be nullable aware.
+Setting the project-level nullable context to "enable" could result in many warnings that a user is unable to fix.
+To support this scenario any syntax tree that is determined to be generated will have its nullable state implicitly set to "disable", regardless of the overall project state.
+
+A syntax tree is determined to be generated if meets one or more of the following criteria:
+- File name begins with:
+    - TemporaryGeneratedFile_
+- File name ends with:
+    - .designer.cs
+    - .generated.cs
+    - .g.cs
+    - .g.i.cs
+- Contains a top level comment that contains
+    - `<autogenerated`
+    - `<auto-generateed`
+
+Newer, nullable-aware generators, may then opt-in to nullable analysis by including a generated `#nullable enable` at the beginning of the generated code.
+
+Within the context of a generated syntax tree `#nullable restore` will always set the state to "disable".
+
 ## Public APIs
 There are a few questions that an API consumer would want to answer:
 1. should I print a `?` after the type?

eng/build-utils.ps1

@@ -8,6 +8,7 @@ $ErrorActionPreference="Stop"
 
 $VSSetupDir = Join-Path $ArtifactsDir "VSSetup\$configuration"
 $PackagesDir = Join-Path $ArtifactsDir "packages\$configuration"
+$PublishDataUrl = "https://raw.githubusercontent.com/dotnet/roslyn/master/eng/config/PublishData.json"
 
 $binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $false }
 $nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { $false }
@@ -22,6 +23,37 @@ function GetProjectOutputBinary([string]$fileName, [string]$projectName = "", [s
   return Join-Path $ArtifactsDir "bin\$projectName\$configuration\$tfm\$ridDir$publishDir$fileName"
 }
 
+function GetPublishData() {
+  if (Test-Path variable:global:_PublishData) {
+    return $global:_PublishData
+  }
+
+  Write-Host "Downloading $PublishDataUrl"
+  $content = (Invoke-WebRequest -Uri $PublishDataUrl -UseBasicParsing).Content
+
+  return $global:_PublishData = ConvertFrom-Json $content
+}
+
+function GetBranchPublishData([string]$branchName) {
+  $data = GetPublishData
+
+  if (Get-Member -InputObject $data.branches -Name $branchName) {
+    return $data.branches.$branchName
+  } else {
+    return $null
+  }
+}
+
+function GetReleasePublishData([string]$releaseName) {
+  $data = GetPublishData
+
+  if (Get-Member -InputObject $data.releases -Name $releaseName) {
+    return $data.releases.$releaseName
+  } else {
+    return $null
+  }
+}
+
 # Handy function for executing a command in powershell and throwing if it 
 # fails.
 #

eng/build.ps1

@@ -46,6 +46,9 @@ param (
     [string]$officialBuildId = "",
     [string]$officialSkipApplyOptimizationData = "",
     [string]$officialSkipTests = "",
+    [string]$officialSourceBranchName = "",
+    [string]$officialIbcSourceBranchName = "",
+    [string]$officialIbcDropId = "",
 
     # Test actions
     [switch]$test32,
@@ -100,6 +103,11 @@ function Print-Usage() {
     Write-Host "  -officialBuildId                            An official build id, e.g. 20190102.3"
     Write-Host "  -officialSkipTests <bool>                   Pass 'true' to not run tests"
     Write-Host "  -officialSkipApplyOptimizationData <bool>   Pass 'true' to not apply optimization data"
+    Write-Host "  -officialSourceBranchName <string>          The source branch name"
+    Write-Host "  -officialIbcDropId <string>                 IBC data drop to use (e.g. '20190210.1/935479/1')."
+    Write-Host "                                              'default' for the most recent available for the branch."
+    Write-Host "  -officialIbcSourceBranchName <string>       IBC source branch (e.g. 'master-vs-deps')"
+    Write-Host "                                              'default' to select branch based on eng/config/PublishData.json."
     Write-Host ""
     Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild."
 }
@@ -133,6 +141,9 @@ function Process-Arguments() {
 
     OfficialBuildOnly "officialSkipTests"
     OfficialBuildOnly "officialSkipApplyOptimizationData"
+    OfficialBuildOnly "officialSourceBranchName"
+    OfficialBuildOnly "officialIbcDropId"
+    OfficialBuildOnly "officialIbcSourceBranchName"
 
     if ($officialBuildId) {
         $script:useGlobalNuGetCache = $false
@@ -192,10 +203,14 @@ function BuildSolution() {
     $toolsetBuildProj = InitializeToolset
     $quietRestore = !$ci
     $testTargetFrameworks = if ($testCoreClr) { "netcoreapp2.1" } else { "" }
+    $ibcSourceBranchName = GetIbcSourceBranchName
+    $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" }
 
-    # Do not set these properties to true explicitly, since that would override values set in projects.
+    # Do not set this property to true explicitly, since that would override values set in projects.
     $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } 
-    $suppressPartialNgenOptimization = if (!$applyOptimizationData) { "/p:ApplyPartialNgenOptimization=false" } else { "" }
+
+    # Workaround for some machines in the AzDO pool not allowing long paths (%5c is msbuild escaped backslash)
+    $ibcDir = Join-Path $RepoRoot ".o%5c"
 
     # Setting /p:TreatWarningsAsErrors=true is a workaround for https://github.com/Microsoft/msbuild/issues/3062.
     # We don't pass /warnaserror to msbuild ($warnAsError is set to $false by default above), but set 
@@ -221,13 +236,62 @@ function BuildSolution() {
         /p:QuietRestoreBinaryLog=$binaryLog `
         /p:TestTargetFrameworks=$testTargetFrameworks `
         /p:TreatWarningsAsErrors=true `
-        $suppressPartialNgenOptimization `
+        /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName `
+        /p:VisualStudioIbcDropId=$ibcDropId `
+        /p:EnablePartialNgenOptimization=$applyOptimizationData `
+        /p:IbcOptimizationDataDir=$ibcDir `
         $suppressExtensionDeployment `
         @properties
 }
 
+
+# Get the branch that produced the IBC data this build is going to consume.
+# IBC data are only merged in official built, but we want to test some of the logic in CI builds as well.
+function GetIbcSourceBranchName() {
+    if (Test-Path variable:global:_IbcSourceBranchName) {
+      return $global:_IbcSourceBranchName
+    }
+
+    function calculate {
+        $fallback = "master-vs-deps"
+
+        if (!$officialIbcSourceBranchName) {
+            return $fallback
+        }  
+
+        if ($officialIbcSourceBranchName -ne "default") {
+            return $officialIbcSourceBranchName
+        }
+
+        $branchData = GetBranchPublishData $officialSourceBranchName
+        if ($branchData -eq $null) {
+            Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'.  " + `
+                       "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow
+            return $fallback
+        }
+
+        if (Test-Member $branchData "ibcSourceBranch") {
+            return $branchData.ibcSourceBranch 
+        }
+
+        return $officialSourceBranchName
+    }
+
+    return $global:_IbcSourceBranchName = calculate
+}
+
 # Set VSO variables used by MicroBuildBuildVSBootstrapper pipeline task
-function Set-OptProfVariables() {
+function SetVisualStudioBootstrapperBuildArgs() {
+    $branchData = GetBranchPublishData (GetIbcSourceBranchName)
+
+    # VS branch name is e.g. "lab/d16.0stg", "rel/d15.9", "lab/ml", etc.
+    $vsBranchSimpleName = $branchData.vsBranch.Split('/')[-1]
+    $vsMajorVersion = $branchData.vsMajorVersion
+    $vsChannel = "int.$vsBranchSimpleName"
+
+    Write-Host "##vso[task.setvariable variable=VisualStudio.MajorVersion;]$vsMajorVersion"        
+    Write-Host "##vso[task.setvariable variable=VisualStudio.ChannelName;]$vsChannel"
+
     $insertionDir = Join-Path $VSSetupDir "Insertion"
     $manifestList = [string]::Join(',', (Get-ChildItem "$insertionDir\*.vsman"))
     Write-Host "##vso[task.setvariable variable=VisualStudio.SetupManifestList;]$manifestList"
@@ -444,7 +508,7 @@ try {
     }
 
     if ($ci -and $build -and $msbuildEngine -eq "vs") {
-        Set-OptProfVariables
+        SetVisualStudioBootstrapperBuildArgs
     }
 
     if ($testDesktop -or $testVsi -or $testIOperation) {

eng/publish-assets.ps1

@@ -126,11 +126,6 @@ function Publish-Entry($publishData, [switch]$isBranch) {
     exit 0
 }
 
-function Test-Member($obj, [string]$name) { 
-    $value = Get-Member -Name $name -InputObject $obj 
-    return $value -ne $null
-}
-
 try {
     if ($configuration -eq "") {
         Write-Host "Must provide the build configuration with -configuration"
@@ -141,29 +136,28 @@ try {
 
     $dotnet = Ensure-DotnetSdk
 
-    Write-Host "Downloading PublishData.json"
-    $publishFileContent = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/dotnet/roslyn/master/eng/config/PublishData.json" -UseBasicParsing).Content
-    $data = ConvertFrom-Json $publishFileContent
-
     if ($branchName -ne "" -and $releaseName -ne "") {
         Write-Host "Can only specify -branchName or -releaseName, not both"
         exit 1
     }
-    elseif ($branchName -ne "") {
-        if (-not (Test-Member $data.branches $branchName)) { 
-            Write-Host "$branchName is not listed for publishing"
+
+    if ($branchName -ne "") {
+        $data = GetBranchPublishData $branchName
+        if ($data -eq $null) {
+            Write-Host "Branch $branchName not listed for publishing."
             exit 0
         }
 
-        Publish-Entry $data.branches.$branchName -isBranch:$true
+        Publish-Entry $data -isBranch:$true
     }
-    elseif ($releaseName -ne "") { 
-        if (-not (Test-Member $data.releases $releaseName)) { 
-           Write-Host "$releaseName is not a valid release"
-           exit 1
+    elseif ($releaseName -ne "") {
+        $data = GetReleasePublishData $releaseName
+        if ($data -eq $null) {
+            Write-Host "Release $releaseName not listed for publishing."
+            exit 1
         }
 
-        Publish-Entry $data.releases.$releaseName -isBranch:$false
+        Publish-Entry $data -isBranch:$false
     }
     else {
         Write-Host "Need to specify -branchName or -releaseName"

eng/test-determinism.ps1

@@ -29,8 +29,11 @@ $script:skipList = @()
 function Run-Build([string]$rootDir, [string]$logFileName) {
     # Clean out the previous run
     Write-Host "Cleaning binaries"
+    $stopWatch = [System.Diagnostics.StopWatch]::StartNew()
     Remove-Item -Recurse (Get-BinDir $rootDir) 
     Remove-Item -Recurse (Get-ObjDir $rootDir) 
+    $stopWatch.Stop()
+    Write-Host "Cleaning took $($stopWatch.Elapsed)"
 
     $solution = Join-Path $rootDir "Roslyn.sln"
 
@@ -90,13 +93,16 @@ function Get-FilesToProcess([string]$rootDir) {
 
 # This will build up the map of all of the binaries and their respective hashes.
 function Record-Binaries([string]$rootDir) {
+    $stopWatch = [System.Diagnostics.StopWatch]::StartNew()
     Write-Host "Recording file hashes"
 
     $map = @{ }
     foreach ($fileData in Get-FilesToProcess $rootDir) { 
         Write-Host "`t$($fileData.FileName) = $($fileData.Hash)"
         $map[$fileData.FileId] = $fileData
     }
+    $stopWatch.Stop()
+    Write-Host "Recording took $($stopWatch.Elapsed)"
     return $map
 }
 
@@ -140,6 +146,7 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFileName) {
     $allGood = $true
 
     Write-Host "Testing the binaries"
+    $stopWatch = [System.Diagnostics.StopWatch]::StartNew()
     foreach ($fileData in Get-FilesToProcess $rootDir) {
         $fileId = $fileData.FileId
         $fileName = $fileData.FileName
@@ -188,6 +195,9 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFileName) {
         Write-Host "Please send $zipFile to compiler team for analysis"
         exit 1
     }
+
+    $stopWatch.Stop()
+    Write-Host "Testing took $($stopWatch.Elapsed)"
 }
 
 function Run-Test() {

global.json

@@ -7,6 +7,6 @@
     "xcopy-msbuild": "15.9.0-alpha"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19104.18"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19113.4"
   }
 }