Detect and mark as reference assembly

NuGet.config

@@ -21,6 +21,8 @@
     <add key="dotnet7-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7-transport/nuget/v3/index.json" />
     <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
     <add key="dotnet8-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8-transport/nuget/v3/index.json" />
+    <add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
+    <add key="dotnet9-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9-transport/nuget/v3/index.json" />
     <add key="dotnet-libraries" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json" />
     <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
     <add key="msbuild-prerelease-with-release-version" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-DotNet-msbuild-Trusted-8ffc3fe3/nuget/v3/index.json" />

README.md

@@ -169,9 +169,9 @@ With development builds, internal NuGet feeds are necessary for some scenarios (
 Do not directly edit the table below. Use https://github.com/dotnet/installer/tree/main/tools/sdk-readme-table-generator to help you generate it. Make sure to run the table generator test and make any changes to the generator along with your changes to the table. Daily servicing builds aren't shown here because they may contain upcoming security fixes. All public servicing builds can be downloaded at http://aka.ms/dotnet-download.
 
 ### Table
-*Note* the 7.0.100 build will be finished internally. Below is the last public version available from that branch but is not fully updated with the final runtime.
+*Note* the 8.0.100 build will be finished internally starting in mid-October.
 
-| Platform | main<br>(9.0.x&nbsp;Runtime) | 8.0.1xx-rc2<br>(8.0-rc2&nbsp;Runtime) | Release/7.0.4xx<br>(7.0.x&nbsp;Runtime) |
+| Platform | main<br>(9.0.x&nbsp;Runtime) | 8.0.1xx<br>(8.0&nbsp;Runtime) | Release/7.0.4xx<br>(7.0.x&nbsp;Runtime) |
 | :--------- | :----------: | :----------: | :----------: |
 | **Windows x64** | [![][win-x64-badge-main]][win-x64-version-main]<br>[Installer][win-x64-installer-main] - [Checksum][win-x64-installer-checksum-main]<br>[zip][win-x64-zip-main] - [Checksum][win-x64-zip-checksum-main] | [![][win-x64-badge-8.0.1XX]][win-x64-version-8.0.1XX]<br>[Installer][win-x64-installer-8.0.1XX] - [Checksum][win-x64-installer-checksum-8.0.1XX]<br>[zip][win-x64-zip-8.0.1XX] - [Checksum][win-x64-zip-checksum-8.0.1XX] | [![][win-x64-badge-7.0.4XX]][win-x64-version-7.0.4XX]<br>[Installer][win-x64-installer-7.0.4XX] - [Checksum][win-x64-installer-checksum-7.0.4XX]<br>[zip][win-x64-zip-7.0.4XX] - [Checksum][win-x64-zip-checksum-7.0.4XX] |
 | **Windows x86** | [![][win-x86-badge-main]][win-x86-version-main]<br>[Installer][win-x86-installer-main] - [Checksum][win-x86-installer-checksum-main]<br>[zip][win-x86-zip-main] - [Checksum][win-x86-zip-checksum-main] | [![][win-x86-badge-8.0.1XX]][win-x86-version-8.0.1XX]<br>[Installer][win-x86-installer-8.0.1XX] - [Checksum][win-x86-installer-checksum-8.0.1XX]<br>[zip][win-x86-zip-8.0.1XX] - [Checksum][win-x86-zip-checksum-8.0.1XX] | [![][win-x86-badge-7.0.4XX]][win-x86-version-7.0.4XX]<br>[Installer][win-x86-installer-7.0.4XX] - [Checksum][win-x86-installer-checksum-7.0.4XX]<br>[zip][win-x86-zip-7.0.4XX] - [Checksum][win-x86-zip-checksum-7.0.4XX] |

eng/Version.Details.xml

@@ -63,11 +63,11 @@
     <Dependency Name="Microsoft.AspNetCore.App.Ref.Internal" Version="8.0.0-rc.1.23414.10" CoherentParentDependency="Microsoft.NET.Sdk">
       <Uri>https://github.com/dotnet/aspnetcore</Uri>
       <Sha>10bc40cf7bec57ecf9d2ffd9a7501b34a28d1f18</Sha>
+      <SourceBuild RepoName="aspnetcore" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.AspNetCore.App.Runtime.win-x64" Version="8.0.0-rc.1.23414.10" CoherentParentDependency="Microsoft.NET.Sdk">
       <Uri>https://github.com/dotnet/aspnetcore</Uri>
       <Sha>10bc40cf7bec57ecf9d2ffd9a7501b34a28d1f18</Sha>
-      <SourceBuild RepoName="aspnetcore" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="VS.Redist.Common.AspNetCore.SharedFramework.x64.8.0" Version="8.0.0-rc.1.23414.10" CoherentParentDependency="Microsoft.NET.Sdk">
       <Uri>https://github.com/dotnet/aspnetcore</Uri>
@@ -185,9 +185,9 @@
       <Sha>850f61abed37b617a41fd59b63a37c284af6801d</Sha>
       <SourceBuild RepoName="deployment-tools" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="8.0.0-alpha.1.23414.2">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="9.0.0-alpha.1.23475.2">
       <Uri>https://github.com/dotnet/source-build-externals</Uri>
-      <Sha>72f5ac04ce80508c7a7d7aaba4ecf7df51c5eca2</Sha>
+      <Sha>e45d334fa3fd29018b70c598eced1938c054884d</Sha>
       <SourceBuild RepoName="source-build-externals" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.symreader" Version="2.1.0-beta.23253.1">
@@ -206,18 +206,18 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23451.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23463.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4665b3d04e1da3796b965c3c3e3b97f55c449a6e</Sha>
+      <Sha>1d451c32dda2314c721adbf8829e1c0cd4e681ff</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="8.0.0-beta.23451.1">
+    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="8.0.0-beta.23463.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4665b3d04e1da3796b965c3c3e3b97f55c449a6e</Sha>
+      <Sha>1d451c32dda2314c721adbf8829e1c0cd4e681ff</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.23451.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="8.0.0-beta.23463.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4665b3d04e1da3796b965c3c3e3b97f55c449a6e</Sha>
+      <Sha>1d451c32dda2314c721adbf8829e1c0cd4e681ff</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Darc" Version="1.1.0-beta.23430.3">
       <Uri>https://github.com/dotnet/arcade-services</Uri>

eng/Versions.props

@@ -39,7 +39,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>8.0.0-beta.23451.1</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>8.0.0-beta.23463.1</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade-services -->

eng/pipelines/source-build-sdk-diff-tests.yml

@@ -26,6 +26,7 @@ jobs:
     targetRid: centos.8-x64
     architecture: x64
     dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
+    includeArtifactsSize: true
 
 - template: templates/jobs/sdk-diff-tests.yml
   parameters:

eng/pipelines/templates/jobs/sdk-diff-tests.yml

@@ -11,6 +11,10 @@ parameters:
 - name: dotnetDotnetRunId
   type: string
 
+- name: includeArtifactsSize
+  type: boolean
+  default: false
+
 jobs:
 - job: ${{ parameters.buildName }}_${{ parameters.architecture }}
   timeoutInMinutes: 150
@@ -51,36 +55,25 @@ jobs:
     env:
       AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
 
-  - task: DownloadPipelineArtifact@2
-    displayName: Download MSFT SDK
-    inputs:
-      buildType: specific
-      buildVersionToDownload: specific
-      project: internal
-      pipeline: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
-      buildId: $(InstallerBuildId)
-      artifact: BlobArtifacts
-      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' 
-      allowPartiallySucceededBuilds: true
-      allowFailedBuilds: true
-      downloadPath: $(Pipeline.Workspace)/Artifacts
-      checkDownloadedFiles: true
-
-  - task: DownloadPipelineArtifact@2
-    displayName: Download Source Build SDK
-    inputs:
-      buildType: specific
-      buildVersionToDownload: specific
-      project: internal
-      pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
-      buildId: $(DotnetDotnetBuildId)
-      artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
+  - template: ../steps/download-pipeline-artifact.yml
+    parameters:
+      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
+      displayName: Download MSFT SDK
+
+  - template: ../steps/download-vmr-artifact.yml
+    parameters:
+      buildName: ${{ parameters.buildName }}
+      architecture: ${{ parameters.architecture }}
       patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz' 
-      allowPartiallySucceededBuilds: true
-      allowFailedBuilds: true
-      downloadPath: $(Pipeline.Workspace)/Artifacts
-      checkDownloadedFiles: true
-
+      displayName: Download Source Build SDK
+
+  - template: ../steps/download-vmr-artifact.yml
+    parameters:
+      buildName: ${{ parameters.buildName }}
+      architecture: ${{ parameters.architecture }}
+      patterns: '**/Private.SourceBuilt.Artifacts.+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*).${{ parameters.targetRid }}.tar.gz' 
+      displayName: Download Source Built Artifacts
+
   - script: |
       msft_sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
 
@@ -96,10 +89,18 @@ jobs:
         exit 1
       fi
 
+      artifacts_path=$(find "$(Pipeline.Workspace)/Artifacts" -name "Private.SourceBuilt.Artifacts.*.${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
+
+      if [[ -z "$artifacts_path" ]]; then
+        echo "Source-build artifacts path does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(DotnetDotnetBuildId) might have failed"
+        exit 1
+      fi
+
       eng/common/build.sh -bl --projects $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --restore
 
       echo "##vso[task.setvariable variable=MsftSdkTarballPath]$(Pipeline.Workspace)/Artifacts/$msft_sdk_tarball_name"
       echo "##vso[task.setvariable variable=SdkTarballPath]$(Pipeline.Workspace)/Artifacts/$sdk_tarball_name"
+      echo "##vso[task.setvariable variable=SourceBuiltArtifactsPath]$(Pipeline.Workspace)/Artifacts/$artifacts_path"
     displayName: Prepare Tests
     workingDirectory: $(Build.SourcesDirectory)
 
@@ -115,12 +116,13 @@ jobs:
       -clp:v=m
       -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=$(MsftSdkTarballPath)
       -e SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath)
-      -e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH= 
+      -e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=$(SourceBuiltArtifactsPath)
       -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=false
       -e SMOKE_TESTS_RUNNING_IN_CI=true
       -e SMOKE_TESTS_TARGET_RID=${{ parameters.targetRid }}
       -e SMOKE_TESTS_PORTABLE_RID=linux-${{ parameters.architecture }}
-      -e SMOKE_TESTS_CUSTOM_PACKAGES_PATH= 
+      -e SMOKE_TESTS_CUSTOM_PACKAGES_PATH=
+      -e SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE=${{ parameters.includeArtifactsSize }}
     displayName: Run Tests
     workingDirectory: $(Build.SourcesDirectory)
 

eng/pipelines/templates/steps/download-pipeline-artifact.yml

@@ -0,0 +1,35 @@
+parameters:
+- name: pipeline
+  type: string
+  default: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
+
+- name: buildId
+  type: string
+  default: $(InstallerBuildId)
+
+- name: artifact
+  type: string
+  default: BlobArtifacts
+
+- name: patterns
+  type: string
+
+- name: displayName
+  type: string
+  default: Download Pipeline Artifact
+
+steps:
+- task: DownloadPipelineArtifact@2
+  inputs:
+    buildType: specific
+    buildVersionToDownload: specific
+    project: internal
+    pipeline: ${{ parameters.pipeline }}
+    buildId: ${{ parameters.buildId }}
+    artifact: ${{ parameters.artifact }}
+    patterns: ${{ parameters.patterns }}
+    allowPartiallySucceededBuilds: true
+    allowFailedBuilds: true
+    downloadPath: $(Pipeline.Workspace)/Artifacts
+    checkDownloadedFiles: true
+  displayName: ${{ parameters.displayName }}

eng/pipelines/templates/steps/download-vmr-artifact.yml

@@ -0,0 +1,22 @@
+parameters:
+- name: buildName
+  type: string
+
+- name: architecture
+  type: string
+
+- name: patterns
+  type: string
+
+- name: displayName
+  type: string
+  default: Download VMR Artifact
+
+steps:
+- template: ../steps/download-pipeline-artifact.yml
+  parameters:
+    pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
+    buildId: $(DotnetDotnetBuildId)
+    artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
+    patterns: ${{ parameters.patterns }}
+    displayName: ${{ parameters.displayName }}

global.json

@@ -11,7 +11,7 @@
     "cmake": "3.21.0"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23451.1",
-    "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.23451.1"
+    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23463.1",
+    "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.23463.1"
   }
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs

@@ -12,6 +12,8 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
 using System.Security.Cryptography;
 using System.Text;
 using System.Xml;
@@ -288,7 +290,14 @@ private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry
                 AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
                 if (IsAssemblyPoisoned(fileToCheck))
                 {
-                    poisonEntry.Type |= PoisonType.AssemblyAttribute;
+                    if (fileToCheck.EndsWith(".dll") && HasAttributeOfType(fileToCheck, "System.Runtime.CompilerServices.ReferenceAssemblyAttribute"))
+                    {
+                        poisonEntry.Type |= PoisonType.ReferenceAssemblyAttribute;
+                    }
+                    else
+                    {
+                        poisonEntry.Type |= PoisonType.AssemblyAttribute;
+                    }
                 }
             }
             catch
@@ -320,6 +329,43 @@ private static bool IsAssemblyPoisoned(string path)
             return false;
         }
 
+        private static bool HasAttributeOfType(string assemblyPath, string desiredAttributeType)
+        {
+            using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+            using var peReader = new PEReader(stream);
+
+            MetadataReader reader = peReader.GetMetadataReader();
+            return reader.CustomAttributes.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
+                    .Any(attr => IsCustomAttributeOfType(reader, attr, desiredAttributeType));
+        }
+
+        private static bool IsCustomAttributeOfType(MetadataReader reader, CustomAttribute attr, string desiredAttributeType)
+        {
+            string customType = "";
+            if (attr.Constructor.Kind == HandleKind.MethodDefinition)
+            {
+                MethodDefinition mdef = reader.GetMethodDefinition((MethodDefinitionHandle)attr.Constructor);
+                TypeDefinition tdef = reader.GetTypeDefinition(mdef.GetDeclaringType());
+                customType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}";
+            }
+            else if (attr.Constructor.Kind == HandleKind.MemberReference)
+            {
+                MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
+
+                if (mref.Parent.Kind == HandleKind.TypeReference)
+                {
+                    TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
+                    customType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
+                }
+                else if (mref.Parent.Kind == HandleKind.TypeDefinition)
+                {
+                    TypeDefinition tdef = reader.GetTypeDefinition((TypeDefinitionHandle)mref.Parent);
+                    customType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}";
+                }
+            }
+            return customType == desiredAttributeType;
+        }
+
         private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue<string> futureFilesToCheck)
         {
             var poisonEntry = new PoisonedFileEntry();

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs

@@ -10,6 +10,7 @@ internal enum PoisonType
         None = 0,
         Hash = 1,
         AssemblyAttribute = 2,
+        ReferenceAssemblyAttribute = 3,
         NupkgFile = 4,
     }
 }