Enable source path mapping for deteministic builds

src/fsharp/FSharp.Build/FSBuild.txt

@@ -1,2 +1,6 @@
 # FSharp.Build resource strings 
 toolpathUnknown,"ToolPath is unknown; specify the path to the tool."
+mapSourceRootsContainsDuplicate,"SourceRoot contains duplicate items '%s' with conflicting metadata '%s': '%s' and '%s'"
+mapSourceRootsPathMustEndWithSlashOrBackslash,"SourceRoot paths are required to end with a slash or backslash: '%s'"
+mapSourceRootsNoTopLevelSourceRoot,"SourceRoot items must include at least one top-level (not nested) item when DeterministicSourcePaths is true"
+mapSourceRootsNoSuchTopLevelSourceRoot,"The value of SourceRoot.ContainingRoot was not found in SourceRoot items, or the corresponding item is not a top-level source root: '%s'"

src/fsharp/FSharp.Build/FSharp.Build.fsproj

@@ -28,6 +28,7 @@
  <Compile Include="WriteCodeFragment.fs" />
  <Compile Include="CreateFSharpManifestResourceName.fs" />
  <Compile Include="SubstituteText.fs" />
+ <Compile Include="MapSourceRoots.fs" />
  <None Include="Microsoft.FSharp.Targets" CopyToOutputDirectory="PreserveNewest" />
  <None Include="Microsoft.Portable.FSharp.Targets" CopyToOutputDirectory="PreserveNewest" />
  <None Include="Microsoft.FSharp.NetSdk.targets" CopyToOutputDirectory="PreserveNewest" />

src/fsharp/FSharp.Build/MapSourceRoots.fs

@@ -0,0 +1,191 @@
+namespace FSharp.Build
+
+open System
+open System.IO
+open Microsoft.Build.Framework
+open Microsoft.Build.Utilities
+open System.Collections.Generic
+
+(*
+ This type is a translation of the matching MapSourceRoots task in Roslyn,
+ which is planned to move to a shared location at some point in the future.
+
+ Until then, this version will be used. The exact source used is:
+ https://github.com/dotnet/roslyn/blob/69d3fb733e6c74a41c118bf905739163cf5aef2a/src/Compilers/Core/MSBuildTask/MapSourceRoots.cs,
+ with matching targets usage at:
+ https://github.com/dotnet/roslyn/blob/69d3fb733e6c74a41c118bf905739163cf5aef2a/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L79-L127
+
+*)
+
+module Utilities =
+ /// <summary>
+ /// Copied from msbuild. ItemSpecs are normalized using this method.
+ /// </summary>
+ let FixFilePath (path: string) =
+ if String.IsNullOrEmpty(path) || Path.DirectorySeparatorChar = '\\'
+ then path
+ else path.Replace('\\', '/');
+
+/// <summary>
+/// Given a list of SourceRoot items produces a list of the same items with added <c>MappedPath</c> metadata that
+/// contains calculated deterministic source path for each SourceRoot.
+/// </summary>
+/// <remarks>
+/// Does not perform any path validation.
+///
+/// The <c>MappedPath</c> is either the path (ItemSpec) itself, when <see cref="Deterministic"/> is false,
+/// or a calculated deterministic source path (starting with prefix '/_/', '/_1/', etc.), otherwise.
+/// </remarks>
+type MapSourceRoots () =
+ inherit Task ()
+
+ static let MappedPath = "MappedPath"
+ static let SourceControl = "SourceControl"
+ static let NestedRoot = "NestedRoot"
+ static let ContainingRoot = "ContainingRoot"
+ static let RevisionId = "RevisionId"
+ static let SourceLinkUrl = "SourceLinkUrl"
+ static let knownMetadataNames =
+ [
+ SourceControl
+ RevisionId
+ NestedRoot
+ ContainingRoot
+ MappedPath
+ SourceLinkUrl
+ ]
+
+ static let (|NullOrEmpty|HasValue|) (s: string) = if String.IsNullOrEmpty s then NullOrEmpty else HasValue s
+ static let ensureEndsWithSlash (path: string) =
+ if path.EndsWith "/"
+ then path
+ else path + "/"
+
+ static let endsWithDirectorySeparator (path: string) =
+ if path.Length = 0
+ then false
+ else
+ let endChar = path.[path.Length - 1]
+ endChar = Path.DirectorySeparatorChar || endChar = Path.AltDirectorySeparatorChar
+
+ static let reportConflictingWellKnownMetadata (log: TaskLoggingHelper) (l: ITaskItem) (r: ITaskItem) =
+ for name in knownMetadataNames do
+ match l.GetMetadata name, r.GetMetadata name with
+ | HasValue lValue, HasValue rValue when lValue <> rValue ->
+ log.LogWarning(FSBuild.SR.mapSourceRootsContainsDuplicate(r.ItemSpec, name, lValue, rValue))
+ | _, _ -> ()
+
+
+ static member PerformMapping (log: TaskLoggingHelper) (sourceRoots: ITaskItem []) deterministic =
+ let mappedSourceRoots = ResizeArray<_>()
+ let rootByItemSpec = Dictionary<string, ITaskItem>();
+
+ for sourceRoot in sourceRoots do
+ // The SourceRoot is required to have a trailing directory separator.
+ // We do not append one implicitly as we do not know which separator to append on Windows.
+ // The usage of SourceRoot might be sensitive to what kind of separator is used (e.g. in SourceLink where it needs
+ // to match the corresponding separators used in paths given to the compiler).
+ if not (endsWithDirectorySeparator sourceRoot.ItemSpec)
+ then
+ log.LogError(FSBuild.SR.mapSourceRootsPathMustEndWithSlashOrBackslash sourceRoot.ItemSpec)
+
+ match rootByItemSpec.TryGetValue sourceRoot.ItemSpec with
+ | true, existingRoot ->
+ reportConflictingWellKnownMetadata log existingRoot sourceRoot
+ sourceRoot.CopyMetadataTo existingRoot
+ | false, _ ->
+ rootByItemSpec.[sourceRoot.ItemSpec] <- sourceRoot
+ mappedSourceRoots.Add sourceRoot
+
+ if log.HasLoggedErrors
+ then None
+ else
+ if deterministic
+ then
+ let topLevelMappedPaths = Dictionary<_,_>()
+ let setTopLevelMappedPaths isSourceControlled =
+
+ let mapNestedRootIfEmpty (root: ITaskItem) =
+ let localPath = root.ItemSpec
+ match root.GetMetadata NestedRoot with
+ | NullOrEmpty ->
+ // root isn't nested
+ if topLevelMappedPaths.ContainsKey(localPath)
+ then
+ log.LogError(FSBuild.SR.mapSourceRootsContainsDuplicate(localPath, NestedRoot, "", ""));
+ else
+ let index = topLevelMappedPaths.Count;
+ let mappedPath = "/_" + (if index = 0 then "" else string index) + "/"
+ topLevelMappedPaths.[localPath] <- mappedPath
+ root.SetMetadata(MappedPath, mappedPath)
+ | HasValue _ -> ()
+
+ for root in mappedSourceRoots do
+ match root.GetMetadata SourceControl with
+ | HasValue v when isSourceControlled -> mapNestedRootIfEmpty root
+ | NullOrEmpty when not isSourceControlled -> mapNestedRootIfEmpty root
+ | _ -> ()
+
+ // assign mapped paths to process source-controlled top-level roots first:
+ setTopLevelMappedPaths true
+
+ // then assign mapped paths to other source-controlled top-level roots:
+ setTopLevelMappedPaths false
+
+ if topLevelMappedPaths.Count = 0
+ then
+ log.LogError(FSBuild.SR.mapSourceRootsNoTopLevelSourceRoot ())
+ else
+ // finally, calculate mapped paths of nested roots:
+ for root in mappedSourceRoots do
+ match root.GetMetadata NestedRoot with
+ | HasValue nestedRoot ->
+ match root.GetMetadata ContainingRoot with
+ | HasValue containingRoot ->
+ // The value of ContainingRoot metadata is a file path that is compared with ItemSpec values of SourceRoot items.
+ // Since the paths in ItemSpec have backslashes replaced with slashes on non-Windows platforms we need to do the same for ContainingRoot.
+ match topLevelMappedPaths.TryGetValue(Utilities.FixFilePath(containingRoot)) with
+ | true, mappedTopLevelPath ->
+ root.SetMetadata(MappedPath, mappedTopLevelPath + ensureEndsWithSlash(nestedRoot.Replace('\\', '/')));
+ | false, _ ->
+ log.LogError(FSBuild.SR.mapSourceRootsNoSuchTopLevelSourceRoot containingRoot)
+ | NullOrEmpty ->
+ log.LogError(FSBuild.SR.mapSourceRootsNoSuchTopLevelSourceRoot "")
+ | NullOrEmpty -> ()
+ else
+ for root in mappedSourceRoots do
+ root.SetMetadata(MappedPath, root.ItemSpec)
+
+ if log.HasLoggedErrors then None else Some (mappedSourceRoots.ToArray())
+
+
+ /// <summary>
+ /// SourceRoot items with the following optional well-known metadata:
+ /// <list type="bullet">
+ /// <term>SourceControl</term><description>Indicates name of the source control system the source root is tracked by (e.g. Git, TFVC, etc.), if any.</description>
+ /// <term>NestedRoot</term><description>If a value is specified the source root is nested (e.g. git submodule). The value is a path to this root relative to the containing root.</description>
+ /// <term>ContainingRoot</term><description>Identifies another source root item that this source root is nested under.</description>
+ /// </list>
+ /// </summary>
+ [<Required>]
+ member val SourceRoots: ITaskItem[] = null with get, set
+
+ /// <summary>
+ /// True if the mapped paths should be deterministic.
+ /// </summary>
+ member val Deterministic = false with get, set
+
+ /// <summary>
+ /// SourceRoot items with <term>MappedPath</term> metadata set.
+ /// Items listed in <see cref="SourceRoots"/> that have the same ItemSpec will be merged into a single item in this list.
+ /// </summary>
+ [<Output>]
+ member val MappedSourceRoots: ITaskItem[] = null with get, set
+
+ override this.Execute() =
+ match MapSourceRoots.PerformMapping this.Log this.SourceRoots this.Deterministic with
+ | None ->
+ false
+ | Some mappings ->
+ this.MappedSourceRoots <- mappings
+ true

src/fsharp/FSharp.Build/Microsoft.FSharp.NetSdk.targets

@@ -12,7 +12,8 @@ WARNING: DO NOT MODIFY this file unless you are knowledgeable about MSBuild and
 -->
 
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-
+ <UsingTask TaskName="MapSourceRoots" AssemblyFile="$(FSharpBuildAssemblyFile)" />
+
  <PropertyGroup>
  <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
  </PropertyGroup>
@@ -90,7 +91,74 @@ WARNING: DO NOT MODIFY this file unless you are knowledgeable about MSBuild and
  <TfmSpecificPackageFile Include="@(_ResolvedOutputFiles)">
  <PackagePath>$(FSharpToolsDirectory)/$(FSharpDesignTimeProtocol)/%(_ResolvedOutputFiles.NearestTargetFramework)/%(_ResolvedOutputFiles.FileName)%(_ResolvedOutputFiles.Extension)</PackagePath>
  </TfmSpecificPackageFile>
+ </ItemGroup>
+ </Target>
+
+ <!--
+ The following targets and props for Deterministic Build/SourceRoot mapping were copied from Roslyn:
+ https://github.com/dotnet/roslyn/blob/f244a9377cd43685fa56dfa9c9a0b374f4560cd9/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L131-L192
+
+ The associated build task was added to this project as well and directly used below.
+
+ When https://github.com/dotnet/msbuild/issues/5398 is resolved these all should be able to be deleted.
+ -->
+
+ <!--
+ ========================
+ DeterministicSourcePaths
+ ========================
+
+ Unless specified otherwise enable deterministic source root (PathMap) when building deterministically on CI server, but not for local builds.
+ In order for the debugger to find source files when debugging a locally built binary the PDB must contain original, unmapped local paths.
+ -->
+ <PropertyGroup>
+ <DeterministicSourcePaths Condition="'$(DeterministicSourcePaths)' == '' and '$(Deterministic)' == 'true' and '$(ContinuousIntegrationBuild)' == 'true'">true</DeterministicSourcePaths>
+ </PropertyGroup>
 
+ <!--
+ ==========
+ SourceRoot
+ ==========
+ All source files of the project are expected to be located under one of the directories specified by SourceRoot item group.
+ This target collects all SourceRoots from various sources.
+ This target calculates final local path for each SourceRoot and sets SourceRoot.MappedPath metadata accordingly.
+ The final path is a path with deterministic prefix when DeterministicSourcePaths is true, and the original path otherwise.
+ In addition, the target validates and deduplicates the SourceRoot items.
+ InitializeSourceControlInformation is an msbuild target that ensures the SourceRoot items are populated from source control.
+ The target is available only if SourceControlInformationFeatureSupported is true.
+ A consumer of SourceRoot.MappedPath metadata, such as Source Link generator, shall depend on this target. 
+ -->
+
+ <Target Name="InitializeSourceRootMappedPaths"
+ DependsOnTargets="_InitializeSourceRootMappedPathsFromSourceControl"
+ Returns="@(SourceRoot)">
+
+ <ItemGroup Condition="'@(_MappedSourceRoot)' != ''">
+ <_MappedSourceRoot Remove="@(_MappedSourceRoot)" />
+ </ItemGroup>
+
+ <MapSourceRoots SourceRoots="@(SourceRoot)" Deterministic="$(DeterministicSourcePaths)">
+ <Output TaskParameter="MappedSourceRoots" ItemName="_MappedSourceRoot" />
+ </MapSourceRoots>
+
+ <ItemGroup>
+ <SourceRoot Remove="@(SourceRoot)" />
+ <SourceRoot Include="@(_MappedSourceRoot)" />
  </ItemGroup>
  </Target>
+
+ <!-- 
+ Declare that target InitializeSourceRootMappedPaths that populates MappedPaths metadata on SourceRoot items is available.
+ -->
+ <PropertyGroup>
+ <SourceRootMappedPathsFeatureSupported>true</SourceRootMappedPathsFeatureSupported>
+ </PropertyGroup>
+
+ <!-- 
+ If InitializeSourceControlInformation target isn't supported, we just continue without invoking that synchronization target. 
+ We'll proceed with SourceRoot (and other source control properties) provided by the user (or blank).
+ -->
+ <Target Name="_InitializeSourceRootMappedPathsFromSourceControl"
+ DependsOnTargets="InitializeSourceControlInformation"
+ Condition="'$(SourceControlInformationFeatureSupported)' == 'true'" />
 </Project>

src/fsharp/FSharp.Build/xlf/FSBuild.txt.cs.xlf

@@ -2,6 +2,26 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
  <file datatype="xml" source-language="en" target-language="cs" original="../FSBuild.resx">
  <body>
+ <trans-unit id="mapSourceRootsContainsDuplicate">
+ <source>SourceRoot contains duplicate items '{0}' with conflicting metadata '{1}': '{2}' and '{3}'</source>
+ <target state="new">SourceRoot contains duplicate items '{0}' with conflicting metadata '{1}': '{2}' and '{3}'</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsNoSuchTopLevelSourceRoot">
+ <source>The value of SourceRoot.ContainingRoot was not found in SourceRoot items, or the corresponding item is not a top-level source root: '{0}'</source>
+ <target state="new">The value of SourceRoot.ContainingRoot was not found in SourceRoot items, or the corresponding item is not a top-level source root: '{0}'</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsNoTopLevelSourceRoot">
+ <source>SourceRoot items must include at least one top-level (not nested) item when DeterministicSourcePaths is true</source>
+ <target state="new">SourceRoot items must include at least one top-level (not nested) item when DeterministicSourcePaths is true</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsPathMustEndWithSlashOrBackslash">
+ <source>SourceRoot paths are required to end with a slash or backslash: '{0}'</source>
+ <target state="new">SourceRoot paths are required to end with a slash or backslash: '{0}'</target>
+ <note />
+ </trans-unit>
  <trans-unit id="toolpathUnknown">
  <source>ToolPath is unknown; specify the path to the tool.</source>
  <target state="translated">Parametr ToolPath není známý. Zadejte cestu k nástroji.</target>

src/fsharp/FSharp.Build/xlf/FSBuild.txt.de.xlf

@@ -2,6 +2,26 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
  <file datatype="xml" source-language="en" target-language="de" original="../FSBuild.resx">
  <body>
+ <trans-unit id="mapSourceRootsContainsDuplicate">
+ <source>SourceRoot contains duplicate items '{0}' with conflicting metadata '{1}': '{2}' and '{3}'</source>
+ <target state="new">SourceRoot contains duplicate items '{0}' with conflicting metadata '{1}': '{2}' and '{3}'</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsNoSuchTopLevelSourceRoot">
+ <source>The value of SourceRoot.ContainingRoot was not found in SourceRoot items, or the corresponding item is not a top-level source root: '{0}'</source>
+ <target state="new">The value of SourceRoot.ContainingRoot was not found in SourceRoot items, or the corresponding item is not a top-level source root: '{0}'</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsNoTopLevelSourceRoot">
+ <source>SourceRoot items must include at least one top-level (not nested) item when DeterministicSourcePaths is true</source>
+ <target state="new">SourceRoot items must include at least one top-level (not nested) item when DeterministicSourcePaths is true</target>
+ <note />
+ </trans-unit>
+ <trans-unit id="mapSourceRootsPathMustEndWithSlashOrBackslash">
+ <source>SourceRoot paths are required to end with a slash or backslash: '{0}'</source>
+ <target state="new">SourceRoot paths are required to end with a slash or backslash: '{0}'</target>
+ <note />
+ </trans-unit>
  <trans-unit id="toolpathUnknown">
  <source>ToolPath is unknown; specify the path to the tool.</source>
  <target state="translated">ToolPath unbekannt. Geben Sie den Pfad zum Tool an.</target>