From a39f81db65643db790116bbb46601269f493f99d Mon Sep 17 00:00:00 2001 From: Pranav K Date: Sat, 4 Mar 2017 20:27:09 -0800 Subject: [PATCH] Update to MSBuild --- .gitignore | 55 +- Universe.sln | 40 + _use-release-management.shade | 243 ---- build-template/NuGet.master.config | 8 - build/PublishPackages.targets | 9 + build/Repositories.props | 58 + build/RepositoryBuild.targets | 69 ++ build/repo.props | 6 - build/repo.targets | 209 +++- makefile.shade | 1186 ------------------- tools/AddCommitInfo/AddCommitInfo.csproj | 12 + tools/AddCommitInfo/Program.cs | 104 ++ tools/BuildGraph/BuildGraph.csproj | 16 + tools/BuildGraph/DGMLFormatter.cs | 56 + tools/BuildGraph/GraphBuilder.cs | 37 + tools/BuildGraph/GraphFormatter.cs | 9 + tools/BuildGraph/GraphNode.cs | 15 + tools/BuildGraph/MSBuildGraphFormatter.cs | 23 + tools/BuildGraph/Program.cs | 81 ++ tools/BuildGraph/Project.cs | 21 + tools/BuildGraph/Repository.cs | 125 ++ tools/BuildGraph/TopologicalSort.cs | 43 + tools/PinVersion.sln | 22 - tools/PinVersion/PinVersion.xproj | 18 - tools/PinVersion/Program.cs | 125 -- tools/PinVersion/project.json | 13 - tools/PinVersions/PinVersionUtility.cs | 139 +++ tools/PinVersions/PinVersions.csproj | 16 + tools/PinVersions/Program.cs | 53 + tools/shared/DependencyGraphSpecProvider.cs | 92 ++ tools/shared/DotNetMuxer.cs | 57 + 31 files changed, 1252 insertions(+), 1708 deletions(-) create mode 100644 Universe.sln delete mode 100644 _use-release-management.shade delete mode 100644 build-template/NuGet.master.config create mode 100644 build/PublishPackages.targets create mode 100644 build/Repositories.props create mode 100644 build/RepositoryBuild.targets delete mode 100644 build/repo.props delete mode 100644 makefile.shade create mode 100644 tools/AddCommitInfo/AddCommitInfo.csproj create mode 100644 tools/AddCommitInfo/Program.cs create mode 100644 tools/BuildGraph/BuildGraph.csproj create mode 100644 tools/BuildGraph/DGMLFormatter.cs create mode 100644 tools/BuildGraph/GraphBuilder.cs create mode 100644 tools/BuildGraph/GraphFormatter.cs create mode 100644 tools/BuildGraph/GraphNode.cs create mode 100644 tools/BuildGraph/MSBuildGraphFormatter.cs create mode 100644 tools/BuildGraph/Program.cs create mode 100644 tools/BuildGraph/Project.cs create mode 100644 tools/BuildGraph/Repository.cs create mode 100644 tools/BuildGraph/TopologicalSort.cs delete mode 100644 tools/PinVersion.sln delete mode 100644 tools/PinVersion/PinVersion.xproj delete mode 100644 tools/PinVersion/Program.cs delete mode 100644 tools/PinVersion/project.json create mode 100644 tools/PinVersions/PinVersionUtility.cs create mode 100644 tools/PinVersions/PinVersions.csproj create mode 100644 tools/PinVersions/Program.cs create mode 100644 tools/shared/DependencyGraphSpecProvider.cs create mode 100644 tools/shared/DotNetMuxer.cs diff --git a/.gitignore b/.gitignore index a2e5dcfcf..8d0665501 100644 --- a/.gitignore +++ b/.gitignore @@ -14,56 +14,7 @@ artifacts StyleCop.Cache node_modules *.snk -.nuget/NuGet.exe -project.lock.json .build -.nuget/ - -# repo folders -/Antiforgery/ -/AzureIntegration/ -/BasicMiddleware/ -/BrowserLink/ -/CORS/ -/Caching/ -/Common/ -/Configuration/ -/DataProtection/ -/DependencyInjection/ -/Diagnostics/ -/DotNetTools/ -/EntityFramework.Tools/ -/EntityFramework/ -/Entropy/ -/EventNotification/ -/FileSystem/ -/Hosting/ -/HtmlAbstractions/ -/HttpAbstractions/ -/HttpSysServer/ -/IISIntegration/ -/Identity/ -/IdentityService/ -/JsonPatch/ -/KestrelHttpServer/ -/Localization/ -/Logging/ -/MetaPackages/ -/Microsoft.Data.Sqlite/ -/MusicStore/ -/Mvc/ -/MvcPrecompilation/ -/Options/ -/PlatformAbstractions/ -/Proxy/ -/Razor/ -/ResponseCaching/ -/Routing/ -/Scaffolding/ -/Security/ -/ServerTests/ -/Session/ -/SignalR/ -/StaticFiles/ -/Testing/ -/WebSockets/ +.nuget +.r +.deps \ No newline at end of file diff --git a/Universe.sln b/Universe.sln new file mode 100644 index 000000000..26fc64777 --- /dev/null +++ b/Universe.sln @@ -0,0 +1,40 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26228.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BuildGraph", "tools\BuildGraph\BuildGraph.csproj", "{B0621D49-4770-4552-9425-D6BD2CF0FB50}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AddCommitInfo", "tools\AddCommitInfo\AddCommitInfo.csproj", "{0DC180DF-D87B-448A-BCDE-2A648F41103E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PinVersions", "tools\PinVersions\PinVersions.csproj", "{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{085280EC-7055-426A-BF9C-1B692B9599AB}" + ProjectSection(SolutionItems) = preProject + tools\shared\DependencyGraphSpecProvider.cs = tools\shared\DependencyGraphSpecProvider.cs + tools\shared\DotNetMuxer.cs = tools\shared\DotNetMuxer.cs + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B0621D49-4770-4552-9425-D6BD2CF0FB50}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B0621D49-4770-4552-9425-D6BD2CF0FB50}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B0621D49-4770-4552-9425-D6BD2CF0FB50}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B0621D49-4770-4552-9425-D6BD2CF0FB50}.Release|Any CPU.Build.0 = Release|Any CPU + {0DC180DF-D87B-448A-BCDE-2A648F41103E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0DC180DF-D87B-448A-BCDE-2A648F41103E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0DC180DF-D87B-448A-BCDE-2A648F41103E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0DC180DF-D87B-448A-BCDE-2A648F41103E}.Release|Any CPU.Build.0 = Release|Any CPU + {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/_use-release-management.shade b/_use-release-management.shade deleted file mode 100644 index 83c3c4bb4..000000000 --- a/_use-release-management.shade +++ /dev/null @@ -1,243 +0,0 @@ -#update-release - -// Merge dev branch to release - @{ - Parallel.ForEach(GetAllRepos(), CloneOrUpdate); - - Log.Info("************************************* Checking repos for diffs *************************"); - - foreach (var repo in GetAllRepos()) - { - Log.Info("Checking repo: " + repo); - // Check if the repo previously had a release branch - try - { - GitCommand(repo, "rev-parse --verify --quiet origin/release"); - } - catch - { - Log.Info("Repository " + repo + " does not have a release branch."); - continue; - } - - try - { - GitCommand(repo, "log -1 --exit-code origin/dev..origin/release"); - } - catch - { - Log.Warn("Unmerged changes in repository " + repo); - GitCommand(repo, "log origin/dev..origin/release"); - throw; - } - } - - Log.Info("No conflicts in repos, continuing with creating release branch."); - - foreach (var repo in GetAllRepos()) - { - GitCommand(repo, "checkout origin/dev -B release"); - var filesChanged = false; - - // Update NuGet.Config - var nugetConfigPath = Path.Combine(repo, "NuGet.config"); - if (File.Exists(nugetConfigPath)) - { - var original = File.ReadAllText(nugetConfigPath); - var modified = original - .Replace("https://dotnet.myget.org/F/aspnetcore-ci-dev", "https://dotnet.myget.org/F/aspnetcore-ci-release"); - - if (!string.Equals(original, modified, StringComparison.Ordinal)) - { - File.WriteAllText(nugetConfigPath, modified); - GitCommand(repo, "add NuGet.config"); - filesChanged = true; - } - } - - var buildPs1Path = Path.Combine(repo, "build.ps1"); - if (File.Exists(buildPs1Path)) - { - var original = File.ReadAllText(buildPs1Path); - var modified = original - .Replace("https://github.com/aspnet/KoreBuild/archive/dev.zip", "https://github.com/aspnet/KoreBuild/archive/release.zip"); - - if (!string.Equals(original, modified, StringComparison.Ordinal)) - { - File.WriteAllText(buildPs1Path, modified); - GitCommand(repo, "add build.ps1"); - filesChanged = true; - } - } - - var buildShPath = Path.Combine(repo, "build.sh"); - if (File.Exists(buildShPath)) - { - var original = File.ReadAllText(buildShPath); - var modified = original - .Replace("https://github.com/aspnet/KoreBuild/archive/dev.zip", "https://github.com/aspnet/KoreBuild/archive/release.zip"); - - if (!string.Equals(original, modified, StringComparison.Ordinal)) - { - File.WriteAllText(buildShPath, modified); - GitCommand(repo, "add build.sh"); - filesChanged = true; - } - } - - if (filesChanged) - { - GitCommand(repo, "commit -m \"Updating to release.\""); - } - - GitCommand(repo, "push origin release:release"); - GitCommand(repo, "checkout origin/dev -B dev"); - GitCommand(repo, "merge release -s ours"); - GitCommand(repo, "push origin dev:dev"); - } - } - -#update-master .pull-all - -// Merge release branch to master - -// Pin versions of packages in project.json and updated project.lock.json - -// More information https://github.com/aspnet/Universe/wiki/%23pin-version-:-Pinning-package-version-for-a-particular-release-in-project.json - @{ - var koreBuildTag = GetEnvironmentParameter("KOREBUILD_TAG"); - var coherenceFeed = GetEnvironmentParameter("COHERENCE_FEED"); - - if (string.IsNullOrEmpty(coherenceFeed)) - { - throw new Exception("COHERENCE_FEED not specified. Usually this is Packages-NoTimestamp directory of Coherence-Signed."); - } - - if (string.IsNullOrEmpty(koreBuildTag)) - { - throw new Exception("KOREBUILD_TAG environment variable is not specified."); - } - - var excludeReposForJson = new[] - { - "Coherence", - "Coherence-Signed", - "dnvm", - "Entropy", - "Setup", - "libuv-build", - }; - - Exec("dotnet", "restore", "tools/PinVersion"); - - foreach (var repo in GetAllRepos()) - { - GitCommand(repo, "checkout origin/release -B master"); - - if (File.Exists(Path.Combine(repo, "NuGet.config"))) - { - File.Copy(Path.Combine("build-template", "NuGet.master.config"), - Path.Combine(repo, "NuGet.config"), - overwrite: true); - GitCommand(repo, "add NuGet.*"); - GitCommand(repo, "commit -m \"Updating NuGet.config\""); - } - } - - var reposToPin = GetAllRepos().Except(excludeReposForJson); - var repositoryNamesFile = Path.GetTempFileName(); - - File.WriteAllLines(repositoryNamesFile, reposToPin.Select(r => Path.Combine(Directory.GetCurrentDirectory(), r))); - var pinToolsArgs = string.Format(@"run -p tools/PinVersion ""{0}"" ""{1}"" ""{2}""", - coherenceFeed, - koreBuildTag, - repositoryNamesFile); - Exec("dotnet", pinToolsArgs, ""); - try - { - File.Delete(repositoryNamesFile); - } - catch - { - } - - foreach (var repo in reposToPin) - { - var repoPath = Path.Combine(Directory.GetCurrentDirectory(), repo); - try - { - GitCommand(repo, "commit -am \"Updating json files to pin versions and build files to pin KoreBuild\""); - } - catch - { - // Don't fail if there was nothing to add. - } - } - - foreach (var repo in GetAllRepos()) - { - GitCommand(repo, "push origin +master:master"); - } - - CallTarget("update-prerelease-tags"); - } - -#update-prerelease-tags - -// Update tags on each repo to have the latest release tag - @{ - var preReleaseTag = GetEnvironmentParameter("PRERELEASETAG"); - if (string.IsNullOrEmpty(preReleaseTag)) - { - throw new Exception("PRERELEASETAG tag not defined"); - } - - var versionFile = "version.txt"; - - foreach (var repo in GetAllRepos()) - { - GitCommand(repo, "pull --tags"); - string version = null; - - try - { - GitCommand(repo, string.Format("describe --tags > ..\\{0}", versionFile)); - } - catch - { - version = "1.0.0-" + preReleaseTag; - Log.Warn(string.Format("{0} repo not tagged. Using default version {1}.", repo, version)); - } - - if (version == null) - { - version = File.ReadAllText(versionFile); - File.Delete(versionFile); - } - - Log.Info(string.Format("Current version on repo {0} is {1}", repo, version)); - - var majorVersion = version.Split(new string[]{"-"}, StringSplitOptions.None)[0]; - - var newVersion = majorVersion + string.Format("-{0}", preReleaseTag); - - Log.Info(string.Format("New version for repo is {0}", newVersion)); - - GitCommand(repo, string.Format("tag -f -a {0} -m \"Tag for new release {0}\"", newVersion)); - - GitCommand(repo, "push origin --tags +" + newVersion); - } - } - -functions - @{ - IEnumerable GetAllRepos() - { - var nonDefaultRepos = new[] - { - "CoreCLR", - "Coherence", - "Coherence-Signed", - "KoreBuild", - "Universe", - "libuv-build", - }; - - return Enumerable.Concat(nonDefaultRepos, repositories); - } - } \ No newline at end of file diff --git a/build-template/NuGet.master.config b/build-template/NuGet.master.config deleted file mode 100644 index adbb3c171..000000000 --- a/build-template/NuGet.master.config +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/build/PublishPackages.targets b/build/PublishPackages.targets new file mode 100644 index 000000000..1564ec5a8 --- /dev/null +++ b/build/PublishPackages.targets @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/build/Repositories.props b/build/Repositories.props new file mode 100644 index 000000000..3316f58d6 --- /dev/null +++ b/build/Repositories.props @@ -0,0 +1,58 @@ + + + + dev + git@github.com:aspnet/%(Identity) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build/RepositoryBuild.targets b/build/RepositoryBuild.targets new file mode 100644 index 000000000..2edc30fa4 --- /dev/null +++ b/build/RepositoryBuild.targets @@ -0,0 +1,69 @@ + + + + + + + + RepositoryToBuild=%(RepositoryToBuildInOrder.Identity); + BuildRepositoryRoot=$(_CloneRepositoryRoot)%(RepositoryToBuildInOrder.Identity)\ + + %(RepositoryToBuildInOrder.Order) + + + + + true + false + + + + + + + + build.sh + build.cmd + $(BuildRepositoryRoot)artifacts\build\ + + + + + + + + + + + + + + + + + + + $(RepositoryRoot)tools\PinVersions\bin\$(Configuration)\netcoreapp1.1\PinVersions.dll + $(DotNetPath) exec $(PinToolBinary) --graph-specs-root "$(_RestoreGraphSpecsDirectory) " -s "$(BuildDir) " "$(BuildRepositoryRoot) " + $(PinVersionArgs) -s "$(_DependencyPackagesDirectory) " + + + + + \ No newline at end of file diff --git a/build/repo.props b/build/repo.props deleted file mode 100644 index 38d3132ef..000000000 --- a/build/repo.props +++ /dev/null @@ -1,6 +0,0 @@ - - - true - true - - diff --git a/build/repo.targets b/build/repo.targets index c88777b22..35540504a 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,56 +1,195 @@ - <_SakeTargets Condition="'$(Configuration)' == 'Release'">--config-release + https://dotnet.myget.org/F/aspnetcore-ci-dev/api/v2/package + https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package + + <_BuildGraphFile>$(BuildDir)BuildGraph.proj + <_CloneRepositoryRoot>$(RepositoryRoot).r\ + <_DependencyPackagesDirectory>$(RepositoryRoot).deps\build\ + <_CommitsFile>$(RepositoryRoot).deps\build\RepositoryCommits.proj + <_RestoreGraphSpecsDirectory>$(RepositoryRoot)obj\package-specs\ + + <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'=='' AND '$(CompileOnly)'=='true'">/t:Package /t:VerifyPackages + <_RepositoryBuildTargets Condition="'$(_RepositoryBuildTargets)'==''">/t:Verify + <_RepositoriesWithCommitsFile>$(RepositoryRoot).deps\build\Repositories.props + + $(PrepareDependsOn);FilterRepositories + $(CleanDependsOn);CleanUniverseArtifacts + $(BuildDependsOn);CloneRepositories;BuildRepositories - - - + + + + + + + + - - - <_SakeTargets>$(_SakeTargets):verify-all - - + + + <_RepositoriesToInclude Include="$(KOREBUILD_REPOSITORY_INCLUDE)" /> + + + + + + + + + + + - + + + <_CloneRepository Include="$(MSBuildProjectFullPath)"> + + CloneRepository=%(Repository.Identity); + CloneUrl=%(Repository.CloneUrl); + CloneBranch=%(Repository.Branch); + CloneRepositoryCommit=%(Repository.Commit); + ShallowClone=true; + UseGateBranch=$(UseGateBranch) + + + + + + + + + + + - <_SakeTargets>$(_SakeTargets):ci-build + false + <_CloneArguments>git clone --quiet $(CloneUrl) + <_CloneArguments Condition="'$(ShallowClone)'=='true'">$(_CloneArguments) --depth 1 - + + + + + + + + + + + - - - <_SakeTargets>$(_SakeTargets):ci-test - - + + + + + - - + + + + %(Repository.Identity) + + + + RestoreGraphOutputPath=$(_RestoreGraphSpecsDirectory)%(Solution.Repository)\%(Solution.FileName)%(Solution.Extension).json + + + + - - + + - - + + + + + + + $(BuildDir)Repositories.props + + + + + + + + + + + diff --git a/makefile.shade b/makefile.shade deleted file mode 100644 index bbdd72ab5..000000000 --- a/makefile.shade +++ /dev/null @@ -1,1186 +0,0 @@ - -var VERSION='0.2.1' - -use-release-management -use namespace='System' -use namespace='System.IO' -use namespace='System.Collections.Concurrent' -use namespace='System.Collections.Generic' -use namespace='System.Net' -use namespace='System.Linq' -use namespace='System.Text' -use namespace='System.Text.RegularExpressions' -use namespace='System.Threading.Tasks' -use import="EnvironmentParameters" -use import="Files" -use import="Json" - -functions - @{ - const string DefaultBuildBranch = "dev"; - const string DefaultCoherenceCacheDir = ".coherence-cache"; - const string UniverseCommitsFileName = "commits-universe"; - const string CoherenceCacheVersionFileName = "coherence-version"; - - private static bool Quiet { get; set; } - - static string baseDir = Directory.GetCurrentDirectory(); - static string targetDir = Path.Combine(baseDir, "artifacts", "build"); - - static string buildBranch = GetEnvironmentParameter("BUILD_BRANCH") ?? DefaultBuildBranch; - static string coherencePath = GetEnvironmentParameter("COHERENCE_PATH"); - static string dropsShare = GetEnvironmentParameter("ASPNETCI_DROPS_SHARE", DefaultDropsShare); - static string ciVolatileShare = GetEnvironmentParameter("CI_VOLATILE_SHARE"); - static string koreBuildTargets = GetEnvironmentParameter("KOREBUILD_BUILD_TARGETS"); - static string nugetVersion = GetEnvironmentParameter("NUGET_VERSION"); - static string nugetExe = GetEnvironmentParameter("PUSH_NUGET_EXE"); - static string universeCommitsFile = GetEnvironmentParameter("UNIVERSE_COMMITS_FILE"); - static bool skipNoCredentials = GetEnvironmentParameter("UNIVERSE_SKIP_NO_CREDENTIALS", value => value == "1"); - static string repositoryInclude = GetEnvironmentParameter("KOREBUILD_REPOSITORY_INCLUDE"); - static string repositoryExclude = GetEnvironmentParameter("KOREBUILD_REPOSITORY_EXCLUDE"); - - static string gateBranch = GetEnvironmentParameter("KOREBUILD_GATE_BRANCH"); - private static bool UseGateBranch = !string.IsNullOrEmpty(gateBranch); - - // Doesn't build on Mono since their contracts don't match - string[] repositories = GetRepositoriesToBuild().ToArray(); - - static bool useHttps = UseHttps(baseDir); - static string gitHubUriPrefix = useHttps ? "https://github.com/aspnet/" : "git@github.com:aspnet/"; - } - -var buildTarget = "/t:Compile" -var buildProperties = "/p:Configuration=Debug" - -#default .compile - -#--quiet - @{ - Quiet = true; - } - -#--config-release - @{ - buildProperties = "/p:Configuration=Release"; - } - -#pull -#compile .pull -#install .pull - -#pack - directory create='${targetDir}' - -#pack-install .pack - nuget-local-publish sourcePackagesDir='${targetDir}' - -#git-pull target='pull' - @{ - Parallel.ForEach(repositories, repo => - { - CloneOrUpdate(repo); - }); - } - -#update - @{ - var errors = new List(); - Parallel.ForEach(repositories, repo => - { - try - { - Git("pull --ff-only", repo); - } - catch - { - errors.Add(repo); - } - }); - - if (errors.Count > 0) - { - Log.Info("Failed to pull: " + string.Join(", ", errors)); - throw new InvalidOperationException("update failed"); - } - } - -#sync-commits - @{ - if (string.IsNullOrEmpty(universeCommitsFile)) - { - throw new Exception("UNIVERSE_COMMITS_FILE not specified."); - } - - var commitsToSync = GetCommitsToSync(universeCommitsFile); - - Parallel.ForEach(repositories, repo => - { - if (commitsToSync.ContainsKey(repo) && Directory.Exists(repo)) - { - var syncHash = commitsToSync[repo]; - Console.WriteLine(string.Format("Syncing {0} to {1}", repo, syncHash)); - Git("reset --hard " + syncHash, repo); - } - }); - } - -#verify-all .pull .change-default-build-target-to-verify .build-all - -#ci-test-env - @{ - Environment.SetEnvironmentVariable("CustomUniverseBeforeCommonProps", Path.Combine(Directory.GetCurrentDirectory(), "build", "ci-test.common.props")); - } - -#ci-test .ci-test-env .pull .use-local-coherence .sync-commits .remove-src-folders .change-default-build-target-to-verify .build-all - -#ci-pull - @{ - var threads = int.Parse(Environment.GetEnvironmentVariable("UNIVERSE_THREADS") ?? "4"); - - Parallel.ForEach(repositories, new ParallelOptions { MaxDegreeOfParallelism = threads }, repo => - { - var useBuildBranch = true; - if (Directory.Exists(repo)) - { - Directory.Delete(repo, recursive: true); - } - - if (UseGateBranch) - { - try - { - GitCommand("", string.Format("clone --quiet --depth 1 --branch {0} git@github.com:aspnet/{1}.git", gateBranch, repo)); - useBuildBranch = false; - Log.Warn(string.Format( - "{0} has a branch '{1}' which is overriding use of {2}. {1} must be deleted before {2} will be used.", - repo, - gateBranch, - buildBranch)); - } - catch (Exception ex) - { - Log.Info(string.Format("{0} doesn't exist, falling back to {1}.", gateBranch, buildBranch)); - } - } - - if (useBuildBranch) - { - GitCommand("", string.Format("clone --quiet --depth 1 --branch {0} git@github.com:aspnet/{1}.git", buildBranch, repo)); - } - }); - } - -#show-build-graph - @{ - var batchedRepos = GetBuildGraph(); - Log.Info("Building repositories in batches: "); - foreach (var repos in batchedRepos) - { - Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos))); - } - } - -#ci-build - @{ - var nugetExe = Path.Combine(Directory.GetCurrentDirectory(), ".build", "nuget.exe"); - var universeArtifacts = Path.Combine(Directory.GetCurrentDirectory(), "artifacts"); - var universeBuild = Path.Combine(universeArtifacts, "build"); - var packagesPublishDir = Path.Combine(Directory.GetCurrentDirectory(), ".nuget", "publishDir"); - // Historically we've always produced coherent builds using this target. Unless specified otherwise, - // we'll continue to produce coherent builds here. - var createCoherentBuild = Environment.GetEnvironmentVariable("CREATE_COHERENT_BUILD") != "false"; - - // Snapshot the .build folder - if (!IsLinux) - { - Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(universeArtifacts, ".build"), ""); - } - else - { - CopyFolder(".build", Path.Combine(universeArtifacts, ".build"), true); - } - - var commits = new ConcurrentDictionary(); - var threads = int.Parse(Environment.GetEnvironmentVariable("UNIVERSE_THREADS") ?? "4"); - buildTarget = koreBuildTargets ?? "/t:Package /t:VerifyPackages /t:NuGetInstall"; - buildTarget += " " + buildProperties; - - var batchedRepos = GetBuildGraph(); - Log.Info("Building repositories in batches: "); - foreach (var repos in batchedRepos) - { - Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos))); - } - - Directory.CreateDirectory(universeBuild); - if (createCoherentBuild) - { - if (!IsLinux) - { - // Publish to a directory and use that as a package source for later builds. This doesn't work in Linux due to - // https://github.com/NuGet/Home/issues/2383 - - Directory.CreateDirectory(packagesPublishDir); - Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_ARTIFACTS", packagesPublishDir); - Environment.SetEnvironmentVariable("PACKAGES_PUBLISH_DIR", packagesPublishDir); - } - else - { - Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_ARTIFACTS", universeBuild); - } - } - - if (Environment.GetEnvironmentVariable("UNIVERSE_PUSH_TO_VOLATILE") == "true") - { - Environment.SetEnvironmentVariable("NUGET_PUBLISH_FEED", "https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package"); - } - - foreach (var batch in batchedRepos) - { - Parallel.ForEach(batch.ToArray(), new ParallelOptions { MaxDegreeOfParallelism = threads }, repo => - { - var blockName = string.Format("Building {0}", repo); - - if (!IsLinux) - { - Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), ""); - } - else - { - CopyFolder(".build", Path.Combine(repo, ".build"), true); - } - - try - { - Exec(CreateBuildWithFlowId(repo), buildTarget, repo); - var repoArtifacts = Path.Combine(repo, "artifacts"); - var repoBuild = Path.Combine(repoArtifacts, "build"); - if (Directory.Exists(repoBuild)) - { - foreach (var source in Directory.EnumerateFiles(repoBuild, "*.nupkg")) - { - File.Copy(source, Path.Combine(universeBuild, Path.GetFileName(source)), overwrite: true); - } - - var commitFile = Path.Combine(repoArtifacts, "commit"); - if (File.Exists(commitFile)) - { - commits.TryAdd(repo, File.ReadAllLines(commitFile)[0]); - } - - if (!string.IsNullOrEmpty(ciVolatileShare)) - { - Log.Info("Publishing packages to " + ciVolatileShare); - NuGetPackagesAdd(repoBuild, ciVolatileShare); - } - } - - Log.Info(string.Format("Build {0} succeeded", repo)); - } - catch (Exception ex) - { - Log.Error("Building '" + repo + "' failed: " + ex); - throw; - } - }); - } - - var commitsAsString = string.Join("\n", commits.Select(c => c.Key + ":" + c.Value)); - File.WriteAllText(Path.Combine(universeArtifacts, "commits"), commitsAsString); - } - -#remove-src-folders - @{ - var excluded = GetNoSrcDeleteRepositories(); - foreach (var repo in GetRepositoriesToBuild()) - { - if (!excluded.Contains(repo)) - { - Log.Info("Remove src from " + repo); - RemoveSrcFolder(repo); - } - else - { - Log.Info("Keeping the sources for " + repo + " because it's explicitly excluded or it is using VS 2017"); - } - } - } - -#use-local-coherence description='Sets up environment variables to use Coherence at COHERENCE_PATH' - @{ - if (string.IsNullOrEmpty(coherencePath)) - { - throw new Exception("COHERENCE_PATH not specified."); - } - - if (string.IsNullOrEmpty(universeCommitsFile)) - { - universeCommitsFile = Path.Combine(coherencePath, UniverseCommitsFileName); - if (!File.Exists(universeCommitsFile)) - { - throw new Exception("Universe commits file could not be found at " + universeCommitsFile); - } - } - - var coherencePackages = Path.Combine(coherencePath, "packages-expanded"); - if (!Directory.Exists(coherencePackages)) - { - coherencePackages = Path.Combine(coherencePath, "build"); - } - if (!Directory.Exists(coherencePackages)) - { - throw new Exception("Packages directory could not be located under " + coherencePath); - } - - Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_AspNetCore", coherencePackages); - } - -#change-default-build-target-to-verify - - buildTarget = "/t:Verify"; - -#change-default-build-target-for-coherence-build - - buildTarget = koreBuildTargets ?? "/t:Compile /t:NuGetInstall"; - -#init - @{ - var templatePath = Path.Combine(baseDir, "build-template"); - var templateFiles = Files.Include(templatePath + Path.DirectorySeparatorChar + "*.*").Select(Path.GetFileName).ToList(); - - foreach(var repo in repositories) - { - foreach (string fileName in templateFiles) - { - var targetFile = Path.Combine(Directory.GetCurrentDirectory(), repo, fileName); - var sourceFile = Path.Combine(Directory.GetCurrentDirectory(), templatePath, fileName); - - // Don't update the makefile - if (fileName.Equals("makefile.shade", StringComparison.OrdinalIgnoreCase) && File.Exists(targetFile)) - { - continue; - } - - // Don't update extra configuration files - if (fileName.Equals("NuGet.master.config", StringComparison.OrdinalIgnoreCase) || - fileName.Equals("NuGet.release.config", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (!File.Exists(targetFile) || - (File.ReadAllText(sourceFile) != File.ReadAllText(targetFile))) - { - Log.Info("Updating " + fileName + " to " + repo); - File.Copy(sourceFile, targetFile, true); - } - } - } - } - - - -#pull-all - -Parallel.ForEach(GetAllRepos(), CloneOrUpdate); - -#only-compile .build-all - -Log.Warn("only-compile target is deprecated. Use build-all"); - -#build-all target='compile' - @{ - var failed = new Dictionary(); - var skipped = new List(); - var universeArtifacts = Path.Combine("artifacts", "build"); - Directory.CreateDirectory(universeArtifacts); - - foreach(var repo in repositories) - { - var blockName = string.Format("Building {0}", repo); - try - { - Log.Info(blockName); - - if (skipNoCredentials) - { - if (!Directory.Exists(repo)) - { - // The directory was not cloned because the credentials were not specified so don't try to build it - skipped.Add(repo); - Log.Warn(string.Format("The repo {0} does not exist locally and it will not be built.", repo)); - continue; - } - } - - if (IsMono) - { - Exec(Path.Combine(repo, "build.sh"), buildTarget, repo); - } - else - { - Exec("build.cmd", buildTarget, repo); - } - - var repoArtifacts = Path.Combine(repo, "artifacts", "build"); - if (Directory.Exists(repoArtifacts)) - { - foreach (var source in Directory.EnumerateFiles(repoArtifacts, "*.nupkg")) - { - File.Copy(source, Path.Combine(universeArtifacts, Path.GetFileName(source)), overwrite: true); - } - - if (!string.IsNullOrEmpty(ciVolatileShare)) - { - Log.Info("Publishing packages to " + ciVolatileShare); - if (string.IsNullOrEmpty(nugetExe)) - { - Log.Warn("PUSH_NUGET_EXE not specified."); - } - else - { - NuGetPackagesAdd(repoArtifacts, ciVolatileShare); - } - } - } - - Log.Info(string.Format("Build {0} succeeded", repo)); - } - catch (Exception ex) - { - Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message)); - failed[repo] = ex; - } - } - - foreach(var repo in repositories) - { - Exception ex; - if (failed.TryGetValue(repo, out ex)) - { - Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message)); - } - else if (skipped.Contains(repo)) - { - Log.Warn(string.Format("Build {0} skipped", repo)); - } - else - { - Log.Info(string.Format("Build {0} succeeded", repo)); - } - } - - if (failed.Any()) - { - throw new Exception("One or more repos failed to build"); - } - } - -#only-install target='install' - @{ - foreach(var repo in repositories) - { - if (IsMono) - { - Exec(Path.Combine(repo, "build.sh"), "install", repo); - } - else - { - Exec("build.cmd", "install", repo); - } - } - } - -#git-status description='Show status of repos known by Universe' - @{ - foreach(var repo in repositories) - { - GitStatus(repo); - } - } - -#git-clean description='REMOVE ALL CHANGES to the working directory' - @{ - foreach(var repo in repositories) - { - GitClean(repo); - } - } - -macro name='ExecClr' program='string' commandline='string' - exec-clr - -macro name='Git' gitCommand='string' gitFolder='string' - git - -macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string' - git-pull - -macro name='GitClone' gitUri='string' gitBranch='string' - git-clone - -macro name='GitConfig' gitOptionName='string' gitOptionValue='string' gitFolder='string' - git-config - -macro name='GitStatus' gitFolder='string' - git gitCommand='status' - -macro name='GitClean' gitFolder='string' - git gitCommand='clean -xdf' - -macro name='GitCommand' gitFolder='string' gitCommand='string' - git - -macro name='Exec' program='string' commandline='string' workingdir='string' - exec - -macro name='NuGetPackagesAdd' sourcePackagesDir='string' targetPackagesDir='string' - nuget-packages-add - -macro name="CopyFolder" sourceDir='string' outputDir='string' overwrite='bool' - copy - -functions - @{ - static IDictionary GetCommitsToSync(string commitsFile) - { - var commits = new Dictionary(); - - if (string.IsNullOrEmpty(commitsFile)) - { - return commits; - } - - Console.WriteLine("Using commits file: " + commitsFile); - var lines = File.ReadAllLines(commitsFile); - foreach(var line in lines) - { - var parts = line.Split(new string[] {":"}, StringSplitOptions.RemoveEmptyEntries); - commits.Add(parts[0], parts[1]); - } - - return commits; - } - - static bool UseHttps(string directory) - { - var filename = Path.Combine(directory, ".git", "config"); - if (!File.Exists(filename)) - { - Console.WriteLine("Unable to find '{0}' file", filename); - return false; - } - - var url = ReadOriginUrl(filename); - return IsHttpsUrl(url); - } - - // Perform equivalent of `git config remote.origin.url` but directly - // read config file to get value. - static string ReadOriginUrl(string filename) - { - // Subsection portion of configuration name is case-sensitive; rest - // of name is case-insensitive. - var beginOriginSection = new Regex(@"^\[(?i:remote) ""origin""\]\s*$"); - var beginSection = new Regex(@"^\["); - var urlLine = new Regex(@"^\s+url = (\S+)\s*$", RegexOptions.IgnoreCase); - - var inRemoteOriginSection = false; - foreach (var line in File.ReadAllLines(filename)) - { - if (beginOriginSection.IsMatch(line)) - { - inRemoteOriginSection = true; - continue; - } - - if (inRemoteOriginSection) - { - if (beginSection.IsMatch(line)) - { - // Read through the section without finding URL line. - break; - } - - var match = urlLine.Match(line); - if (match.Success && match.Groups.Count == 2 && match.Groups[1].Success) - { - return match.Groups[1].Value; - } - } - } - - Console.WriteLine("Unable to parse '{0}' file", filename); - return null; - } - - static bool IsHttpsUrl(string url) - { - if (string.IsNullOrEmpty(url)) - { - return false; - } - - return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase); - } - - static bool IsAccessible(string key) - { - var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key); - req.Method = "HEAD"; - try - { - using (req.GetResponse()) - { - // intentionally empty - } - } - catch (WebException ex) - { - if (ex.Response != null && - ((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound) - { - return false; - } - - // Ignore any other exception. They should surface as part of git clone with well-known messages. - } - return true; - } - - void RemoveSrcFolder(string repo) - { - var srcDir = Path.Combine(repo, "src"); - if (Directory.Exists(srcDir)) - { - Console.WriteLine("Deleting " + srcDir); - Directory.Delete(srcDir, recursive: true); - } - } - - void CloneOrUpdate(string repo) - { - var repoUrl = gitHubUriPrefix + repo + ".git"; - - if (Directory.Exists(repo)) - { - GitPull(repoUrl, buildBranch, repo); - } - else - { - if (useHttps && - !IsAccessible(repo)) - { - if (skipNoCredentials) - { - // This is used on the XPlat CI. Must return because otherwise git will wait for user - // input and hang the build - Log.Warn(string.Format("The repo at '{0}' is not accesible and it will not be cloned.", repoUrl)); - return; - } - - Log.Warn(string.Format("The repo at '{0}' is not accessible. If you do not have access to this repository, skip the git prompt" + - " for credentials to skip cloning this repository. To avoid this prompt, re-clone the Universe repository over ssh.", - repoUrl)); - } - - try - { - GitClone(repoUrl, buildBranch); - } - catch (Exception ex) - { - Log.Warn(string.Format("Unable to clone repository at '{0}': {1}", repoUrl, ex.Message)); - return; - } - } - } - - string FormatForTeamCity(string input) - { - return input.Replace("|", "||") - .Replace("'", "|'") - .Replace("\r", "|r") - .Replace("\n", "|n") - .Replace("]", "|]"); - } - - static IEnumerable GetRepositoriesToBuild() - { - IEnumerable reposToBuild = new HashSet(StringComparer.OrdinalIgnoreCase) - { - "Microsoft.Data.Sqlite", - "PlatformAbstractions", - "Common", - "JsonPatch", - "FileSystem", - "Configuration", - "DependencyInjection", - "EventNotification", - "Options", - "Logging", - "DotNetTools", - "HtmlAbstractions", - "DataProtection", - "HttpAbstractions", - "Testing", - "Caching", - "Razor", - "Hosting", - "AzureIntegration", - "EntityFramework", - "EntityFramework.Tools", - "HttpSysServer", - "KestrelHttpServer", - "IISIntegration", - "ServerTests", - "ResponseCaching", - "Session", - "CORS", - "Routing", - "StaticFiles", - "Diagnostics", - "Security", - "Antiforgery", - "WebSockets", - "Localization", - "BasicMiddleware", - "Proxy", - "Mvc", - "MvcPrecompilation", - "Identity", - "IdentityService", - "Scaffolding", - "SignalR", - "BrowserLink", - "Entropy", - "MusicStore", - "MetaPackages", - }; - - if (!string.IsNullOrEmpty(repositoryInclude)) - { - reposToBuild = new HashSet( - repositoryInclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries), - StringComparer.OrdinalIgnoreCase); - } - - if (!string.IsNullOrEmpty(repositoryExclude)) - { - var reposToExclude = repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries); - reposToBuild = reposToBuild.Except(reposToExclude); - } - - return reposToBuild.ToList(); - } - - static ISet GetNoSrcDeleteRepositories() - { - var repos = new HashSet(StringComparer.OrdinalIgnoreCase); - - var repositoryExclude = Environment.GetEnvironmentVariable("KOREBUILD_NOSRC_EXCLUDE"); - if (!string.IsNullOrEmpty(repositoryExclude)) - { - foreach (var repo in repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries)) - { - repos.Add(repo); - } - } - - foreach (var repo in GetRepositoriesToBuild()) - { - var solutions = Directory.EnumerateFiles(repo, "*.sln") - .Select(MsBuildSolutionInfo.LoadFromFile) - .Where(sln => sln.IsVs2017); - - if (solutions.Any()) - { - repos.Add(repo); - } - } - - return repos; - } - - static string DefaultDropsShare(string value) - { - return value ?? (Environment.OSVersion.Platform == PlatformID.Unix ? "/aspnetci-drops" : @"\\aspnetci\drops"); - } - - string CreateBuildWithFlowId(string repo) - { - string output; - if (IsLinux) - { - output = Path.Combine(repo, "build.sh"); - } - else - { - output = Path.Combine(repo, "build-with-flow-id.cmd"); - File.WriteAllText( - output, - string.Format("set KOREBUILD_FLOWID=KOREBUILD_{0}& call build.cmd %*", repo)); - } - - return Path.GetFullPath(output); - } - - static IList> GetBuildGraph() - { - var repositoryLookup = new List(); - - Parallel.ForEach(GetRepositoriesToBuild(), repo => - { - repositoryLookup.Add(RepoInfoFactory.Create(repo)); - }); - - foreach (var info in repositoryLookup) - { - foreach (var item in info.DependencyNames) - { - var dependency = repositoryLookup.Find(r => r.RepositoryNames.Contains(item)); - if (dependency != null) - { - info.Dependencies.Add(dependency); - } - } - } - - return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray(); - } - - private class MsBuildSolutionInfo - { - public static MsBuildSolutionInfo LoadFromFile(string file) - { - if (!Path.IsPathRooted(file)) - { - file = Path.Combine(System.IO.Directory.GetCurrentDirectory(), file); - } - - var info = new MsBuildSolutionInfo - { - FilePath = file, - Directory = Path.GetDirectoryName(file) - }; - - var lines = File.ReadAllLines(file); - foreach (var line in lines) - { - var match = Regex.Match(line, @"^VisualStudioVersion = ([\d.]+)\s?$"); - if (match == null || match.Groups.Count < 2) - { - continue; - } - Version version; - if (Version.TryParse(match.Groups[1].Value, out version)) - { - info.VisualStudioVersion = version; - break; - } - } - return info; - } - - public string FilePath { get; private set; } - public string Directory { get; private set; } - - public Version VisualStudioVersion { get; set; } - - public bool IsVs2017 { get { return VisualStudioVersion != null && VisualStudioVersion.Major >= 15; } } - } - - - private class RepoInfoFactory - { - public static RepositoryInfo Create(string repo) - { - var info = new RepositoryInfo { Name = repo }; - ResolveSourcesPackages(info, repo); - ResolveProjectJsonProjects(info, repo); - ResolveMsBuildProjects(info, repo); - info.DependencyNames.ExceptWith(info.RepositoryNames); - return info; - } - - private static void ResolveSourcesPackages(RepositoryInfo info, string repo) - { - var repoDir = Path.Combine(Directory.GetCurrentDirectory(), repo); - var ngpvJson = Path.Combine(repoDir, "NuGetPackageVerifier.json"); - if (File.Exists(ngpvJson)) - { - ResolveFromNuGetPackageVerifier(info, ngpvJson); - } - ResolveSourcesFromFolderConventions(info, repoDir); - } - - private static void ResolveSourcesFromFolderConventions(RepositoryInfo info, string repoDir) - { - var directories = Enumerable.Empty(); - var sharedDir = Path.Combine(repoDir, "shared"); - if (Directory.Exists(sharedDir)) - { - directories = directories.Concat( - Directory.GetDirectories(sharedDir, "*.Sources")); - } - - var srcDir = Path.Combine(repoDir, "src"); - if (Directory.Exists(srcDir)) - { - directories = directories.Concat( - Directory.GetDirectories(Path.Combine(repoDir, "src"), "*.Sources")); - } - - directories = directories.Select(Path.GetFileName) - .Where(m => m != null && m.StartsWith("Microsoft.")); - - foreach (var dir in directories) - { - info.RepositoryNames.Add(dir); - } - } - - private static void ResolveFromNuGetPackageVerifier(RepositoryInfo info, string ngpvJson) - { - JsonObject ngpv; - try - { - ngpv = (JsonObject)Json.Deserialize(File.ReadAllText(ngpvJson)); - } - catch (Exception ex) - { - // usually due to comments - // Console.Error.WriteLine("Failed to parse " + ngpvJson + " : " + ex.Message); - return; - } - - if (ngpv == null || ngpv.Keys == null) - { - return; - } - - foreach (var groupKey in ngpv.Keys) - { - var @group = ngpv.ValueAsJsonObject(groupKey); - var packages = @group.ValueAsJsonObject("packages"); - if (packages == null || packages.Keys == null) - { - continue; - } - - foreach (var pkgKey in packages.Keys) - { - info.RepositoryNames.Add(pkgKey); - } - } - } - - private static void ResolveMsBuildProjects(RepositoryInfo info, string repo) - { - Console.WriteLine("Resolving projects from " + repo); - var solutions = Directory.EnumerateFiles(repo, "*.sln") - .Select(MsBuildSolutionInfo.LoadFromFile) - .Where(sln => sln.IsVs2017); - - foreach (var sln in solutions) - { - ResolveMsBuildProject(info, sln); - } - } - - private static void ResolveMsBuildProject(RepositoryInfo info, MsBuildSolutionInfo solution) - { - var intermediate = Path.Combine(Directory.GetCurrentDirectory(), "obj"); - Directory.CreateDirectory(intermediate); - var dgJson = Path.Combine(intermediate, Path.GetFileName(solution.FilePath) + ".graph.json"); - var psi = new ProcessStartInfo - { - UseShellExecute = false, - FileName = "dotnet", - Arguments = "msbuild \"" + solution.FilePath + "\" /v:q /nologo /t:GenerateRestoreGraphFile \"/p:RestoreGraphOutputPath=" + dgJson + "\"" - }; - var p = Process.Start(psi); - p.WaitForExit(); - if (p.ExitCode != 0) - { - Console.WriteLine("warn: Failed to get restore graph from " + solution.FilePath); - return; - } - - JsonObject graph; - try - { - graph = (JsonObject)Json.Deserialize(File.ReadAllText(dgJson)); - } - catch (Exception ex) - { - Console.Error.WriteLine("Failed to parse dependency graph"); - throw; - } - - var format = graph.ValueAsInt("format"); - if (format != 1) - { - Console.Error.WriteLine("Dependency graph file in unsupported format version: " + format); - throw new FormatException(); - } - - var projects = graph.ValueAsJsonObject("projects"); - foreach (var projectKey in projects.Keys) - { - var project = projects.ValueAsJsonObject(projectKey); - var restore = project.ValueAsJsonObject("restore"); - var projectName = restore.ValueAsString("projectName"); - info.RepositoryNames.Add(projectName); - - var frameworks = project.ValueAsJsonObject("frameworks"); - foreach (var fxKey in frameworks.Keys) - { - var fxDeps = frameworks.ValueAsJsonObject(fxKey).ValueAsJsonObject("dependencies"); - if (fxDeps == null || fxDeps.Keys == null) - { - continue; - } - - foreach (var depKey in fxDeps.Keys) - { - var dep = fxDeps.ValueAsJsonObject(depKey); - if (dep.ValueAsString("target") != "Package") - { - continue; - } - // todo version? - info.DependencyNames.Add(depKey); - } - } - } - } - - private static void ResolveProjectJsonProjects(RepositoryInfo info, string repo) - { - var srcDir = Path.Combine(repo, "src"); - - if (Directory.Exists(srcDir)) - { - foreach (var directory in Directory.EnumerateDirectories(srcDir)) - { - info.RepositoryNames.Add(Path.GetFileName(directory)); - var projectJson = Path.Combine(directory, "project.json"); - if (File.Exists(projectJson)) - { - ResolveDependencies(projectJson, info.DependencyNames); - } - } - } - - var otherDirs = new[] { "test", "samples", "tools" }; - for (var i = 0; i < otherDirs.Length; i++) - { - var otherDir = Path.Combine(repo, otherDirs[i]); - if (Directory.Exists(otherDir)) - { - foreach (var directory in Directory.EnumerateDirectories(otherDir)) - { - var projectJson = Path.Combine(directory, "project.json"); - if (File.Exists(projectJson)) - { - ResolveDependencies(projectJson, info.DependencyNames); - } - else - { - // Look for test\Websites\WebsiteName\project.json - foreach (var subDirectory in Directory.EnumerateDirectories(directory)) - { - projectJson = Path.Combine(subDirectory, "project.json"); - if (File.Exists(projectJson)) - { - ResolveDependencies(projectJson, info.DependencyNames); - } - } - } - } - } - } - } - - private static void ResolveDependencies(string projectJsonPath, HashSet dependencies) - { - JsonObject project; - try - { - project = (JsonObject)Json.Deserialize(File.ReadAllText(projectJsonPath)); - } - catch (Exception ex) - { - Console.Error.WriteLine("Failed to parse project at path " + projectJsonPath + Environment.NewLine + ex.Message); - throw; - } - var dependenciesNode = project.ValueAsJsonObject("dependencies"); - AddKeys(dependencies, dependenciesNode); - - var frameworkNodes = project.ValueAsJsonObject("frameworks"); - if (frameworkNodes != null) - { - foreach (var framework in frameworkNodes.Keys) - { - dependenciesNode = frameworkNodes.ValueAsJsonObject(framework).ValueAsJsonObject("dependencies"); - AddKeys(dependencies, dependenciesNode); - } - } - - AddKeys(dependencies, project.ValueAsJsonObject("tools")); - } - - private static void AddKeys(HashSet target, JsonObject source) - { - if (source != null) - { - foreach (var key in source.Keys) - { - target.Add(key); - } - } - } - } - - private class RepositoryInfo - { - public string Name; - - public HashSet RepositoryNames = new HashSet(StringComparer.OrdinalIgnoreCase); - - public HashSet DependencyNames = new HashSet(StringComparer.OrdinalIgnoreCase); - - public HashSet Dependencies = new HashSet(); - - public int Order - { - get - { - return GetOrder(new List(), this); - } - } - - private static int GetOrder(List visited, RepositoryInfo info) - { - if (visited.Contains(info)) - { - throw new Exception("A cyclic dependency between the following repositories has been detected: " + - string.Join(" -> ", visited)); - } - - visited.Add(info); - - var order = 0; - foreach (var dependency in info.Dependencies) - { - order = Math.Max(order, GetOrder(visited, dependency)); - } - - visited.RemoveAt(visited.Count - 1); - - return order + 1; - } - - public override string ToString() - { - return Name; - } - } - } diff --git a/tools/AddCommitInfo/AddCommitInfo.csproj b/tools/AddCommitInfo/AddCommitInfo.csproj new file mode 100644 index 000000000..3797101e7 --- /dev/null +++ b/tools/AddCommitInfo/AddCommitInfo.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp1.1 + + + + + + + \ No newline at end of file diff --git a/tools/AddCommitInfo/Program.cs b/tools/AddCommitInfo/Program.cs new file mode 100644 index 000000000..7df293e07 --- /dev/null +++ b/tools/AddCommitInfo/Program.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using System.Xml.Linq; +using Microsoft.Extensions.CommandLineUtils; + +namespace AddCommitInfo +{ + class Program + { + private const string HeadContentStart = "ref: refs/heads/"; + private const int CommitShaLength = 40; + + static int Main(string[] args) + { + var app = new CommandLineApplication(); + + var repositoriesRootOption = app.Option("-r|--repositories-root", + "Directory containing repositories to create the file for.", + CommandOptionType.SingleValue); + + var repositoryFileOption = app.Argument("Repository file.", "Repository file to update."); + app.OnExecute(() => + { + if (string.IsNullOrEmpty(repositoryFileOption.Value)) + { + Console.Error.WriteLine("Repository file argument not specified."); + return 1; + } + + if (!repositoriesRootOption.HasValue()) + { + Console.Error.WriteLine($"Option {repositoriesRootOption.Template} must have a value."); + return 1; + } + + Execute(repositoriesRootOption.Value().Trim(), repositoryFileOption.Value); + + return 0; + }); + + return app.Execute(args); + } + + private static void Execute(string repositoriesRoot, string repositoryFile) + { + var document = XDocument.Load(repositoryFile); + foreach (var repositoryElement in document.Root.Element("ItemGroup").Elements("Repository")) + { + var repositoryName = repositoryElement.Attribute("Include").Value; + var repositoryDirectory = Path.Combine(repositoriesRoot, repositoryName); + if (!Directory.Exists(repositoryDirectory)) + { + // Repository does not exist. Possibly due to not being cloned. + continue; + } + + var commitHash = ReadCommitHash(repositoryDirectory); + repositoryElement.Add(new XAttribute("Commit", commitHash)); + } + + File.WriteAllText(repositoryFile, document.Root.ToString()); + } + + private static string ReadCommitHash(string repositoryPath) + { + // Based on https://github.com/aspnet/BuildTools/blob/dev/src/Internal.AspNetCore.BuildTools.Tasks/GetGitCommitInfo.cs + var headFile = Path.Combine(repositoryPath, ".git", "HEAD"); + if (!File.Exists(headFile)) + { + throw new Exception($"Unable to determine active git branch for {repositoryPath}."); + } + + var content = File.ReadAllText(headFile).Trim(); + if (content.StartsWith(HeadContentStart)) + { + return ResolveFromBranch(repositoryPath, content); + } + else if (content.Length == CommitShaLength) + { + return content; + } + + throw new Exception($"Unable to determine active git branch. '.git/HEAD' file in unexpected format: '{content}'."); + } + + private static string ResolveFromBranch(string repositoryPath, string head) + { + var branch = head.Substring(HeadContentStart.Length); + + if (string.IsNullOrEmpty(branch)) + { + throw new Exception("Current branch appears to be empty. Failed to retrieve current branch."); + } + + var branchFile = Path.Combine(repositoryPath, ".git", "refs", "heads", branch); + if (!File.Exists(branchFile)) + { + throw new Exception("Unable to determine current git commit hash"); + } + + return File.ReadAllText(branchFile).Trim(); + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/BuildGraph.csproj b/tools/BuildGraph/BuildGraph.csproj new file mode 100644 index 000000000..6c12eb10f --- /dev/null +++ b/tools/BuildGraph/BuildGraph.csproj @@ -0,0 +1,16 @@ + + + + Exe + netcoreapp1.1 + + + + + + + + + + + \ No newline at end of file diff --git a/tools/BuildGraph/DGMLFormatter.cs b/tools/BuildGraph/DGMLFormatter.cs new file mode 100644 index 000000000..23ea409cb --- /dev/null +++ b/tools/BuildGraph/DGMLFormatter.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace BuildGraph +{ + public class DGMLFormatter : GraphFormatter + { + public override void Format(IList nodes, string outputPath) + { + var xmlns = XNamespace.Get("http://schemas.microsoft.com/vs/2009/dgml"); + var xdoc = new XDocument( + new XElement(xmlns + "DirectedGraph", + new XElement(xmlns + "Nodes", GetNodes(xmlns, nodes).ToArray()), + new XElement(xmlns + "Links", GetLinks(xmlns, nodes).ToArray()), + new XElement(xmlns + "Properties", GetProperties(xmlns).ToArray()))); + + using (var writer = File.OpenWrite(outputPath)) + { + xdoc.Save(writer); + } + } + + private IEnumerable GetLinks(XNamespace xmlns, IEnumerable nodes) + { + foreach (var node in nodes) + { + foreach (var outgoing in node.Outgoing) + { + yield return new XElement(xmlns + "Link", + new XAttribute("Source", node.Repository.Name), + new XAttribute("Target", outgoing.Repository.Name)); + } + } + } + + private IEnumerable GetNodes(XNamespace xmlns, IEnumerable nodes) + { + foreach (var node in nodes) + { + yield return new XElement(xmlns + "Node", + new XAttribute("Id", node.Repository.Name), + new XAttribute("Label", $"{node.Repository.Name}")); + } + } + + private IEnumerable GetProperties(XNamespace xmlns) + { + yield return new XElement(xmlns + "Property", + new XAttribute("Id", "Label"), + new XAttribute("Label", "Label"), + new XAttribute("DataType", "String")); + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/GraphBuilder.cs b/tools/BuildGraph/GraphBuilder.cs new file mode 100644 index 000000000..283a01e27 --- /dev/null +++ b/tools/BuildGraph/GraphBuilder.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BuildGraph +{ + public static class GraphBuilder + { + public static IList Generate(IList repositories) + { + // Build global list of primary projects + var primaryProjects = repositories.SelectMany(c => c.Projects) + .ToDictionary(p => p.Name, StringComparer.OrdinalIgnoreCase); + var graphNodes = repositories.Select(r => new GraphNode { Repository = r }) + .ToDictionary(r => r.Repository); + + foreach (var project in repositories.SelectMany(r => r.AllProjects)) + { + var thisProjectRepositoryNode = graphNodes[project.Repository]; + + foreach (var packageDependency in project.PackageReferences) + { + if (primaryProjects.TryGetValue(packageDependency, out var dependencyProject)) + { + var dependencyRepository = dependencyProject.Repository; + var dependencyNode = graphNodes[dependencyRepository]; + + thisProjectRepositoryNode.Incoming.Add(dependencyNode); + dependencyNode.Outgoing.Add(thisProjectRepositoryNode); + } + } + } + + return graphNodes.Values.ToList(); + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/GraphFormatter.cs b/tools/BuildGraph/GraphFormatter.cs new file mode 100644 index 000000000..878dacd4f --- /dev/null +++ b/tools/BuildGraph/GraphFormatter.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace BuildGraph +{ + public abstract class GraphFormatter + { + public abstract void Format(IList nodes, string outputPath); + } +} \ No newline at end of file diff --git a/tools/BuildGraph/GraphNode.cs b/tools/BuildGraph/GraphNode.cs new file mode 100644 index 000000000..236ff6aee --- /dev/null +++ b/tools/BuildGraph/GraphNode.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Diagnostics; + +namespace BuildGraph +{ + [DebuggerDisplay("{Repository.Name}")] + public class GraphNode + { + public Repository Repository { get; set; } + + public ISet Incoming { get; } = new HashSet(); + + public ISet Outgoing { get; } = new HashSet(); + } +} diff --git a/tools/BuildGraph/MSBuildGraphFormatter.cs b/tools/BuildGraph/MSBuildGraphFormatter.cs new file mode 100644 index 000000000..4867ff217 --- /dev/null +++ b/tools/BuildGraph/MSBuildGraphFormatter.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace BuildGraph +{ + public class MSBuildGraphFormatter : GraphFormatter + { + public override void Format(IList nodes, string outputPath) + { + var sortedNodes = nodes.Select(node => new { Repository = node.Repository, Order = TopologicalSort.GetOrder(node) }) + .OrderBy(item => item.Order); + var projectElement = new XElement("Project", + new XElement("ItemGroup", + sortedNodes.Select(item => new XElement("RepositoryToBuildInOrder", + new XAttribute("Include", item.Repository.Name), + new XAttribute("Order", item.Order))))); + + File.WriteAllText(outputPath, projectElement.ToString()); + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/Program.cs b/tools/BuildGraph/Program.cs new file mode 100644 index 000000000..81d0a44de --- /dev/null +++ b/tools/BuildGraph/Program.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.CommandLineUtils; +using UniverseTools; + +namespace BuildGraph +{ + class Program + { + static int Main(string[] args) + { + var app = new CommandLineApplication(); + var outputTypeOption = app.Option("--output-type", + "Output type of generated graph. Valid values are: msbuild, and dgml.", + CommandOptionType.SingleValue); + + var repositoriesRootOption = app.Option("-r|--repositories-root", + "Directory containing repositories to calculate graph for.", + CommandOptionType.SingleValue); + + var packageSpecsDirectoryOption = app.Option("--graph-specs-root", + "Directory containing package specs. (Optional)", + CommandOptionType.SingleValue); + + var outputPathArgument = app.Argument("Output path", "Output path"); + + app.OnExecute(() => + { + if (!repositoriesRootOption.HasValue()) + { + Console.Error.WriteLine($"Option {repositoriesRootOption.Template} must have a value."); + return 1; + } + + var outputPath = outputPathArgument.Value; + if (string.IsNullOrEmpty(outputPath)) + { + Console.Error.WriteLine($"Output path not specified."); + return 1; + } + + var outputDirectory = Path.GetDirectoryName(outputPath); + Directory.CreateDirectory(outputDirectory); + + var outputType = outputTypeOption.Value() ?? "msbuild"; + + + var graphSpecProvider = packageSpecsDirectoryOption.HasValue() + ? new DependencyGraphSpecProvider(packageSpecsDirectoryOption.Value().Trim()) + : DependencyGraphSpecProvider.Default; + IList repositories; + using (graphSpecProvider) + { + repositories = Repository.ReadAllRepositories(repositoriesRootOption.Value().Trim(), graphSpecProvider); + } + + var graph = GraphBuilder.Generate(repositories); + GraphFormatter formatter; + switch (outputType) + { + case "msbuild": + formatter = new MSBuildGraphFormatter(); + break; + case "dgml": + formatter = new DGMLFormatter(); + break; + default: + app.Error.WriteLine($"Unknown output type: {outputType}."); + return 1; + } + + formatter.Format(graph, outputPathArgument.Value); + + return 0; + }); + + return app.Execute(args); + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/Project.cs b/tools/BuildGraph/Project.cs new file mode 100644 index 000000000..12d3da6fb --- /dev/null +++ b/tools/BuildGraph/Project.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; + +namespace BuildGraph +{ + [DebuggerDisplay("{Name}")] + public class Project + { + public Project(string name) + { + Name = name; + } + + public string Name { get; } + + public Repository Repository { get; set; } + + public IList PackageReferences { get; set; } = Array.Empty(); + } +} \ No newline at end of file diff --git a/tools/BuildGraph/Repository.cs b/tools/BuildGraph/Repository.cs new file mode 100644 index 000000000..821ffc54c --- /dev/null +++ b/tools/BuildGraph/Repository.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using NuGet.LibraryModel; +using NuGet.ProjectModel; +using UniverseTools; + +namespace BuildGraph +{ + [DebuggerDisplay("{Name}")] + public class Repository : IEquatable + { + public Repository(string name) + { + Name = name; + } + + public string Name { get; private set; } + + public IList Projects { get; } = new List(); + + public IList SupportProjects { get; } = new List(); + + public IEnumerable AllProjects => Projects.Concat(SupportProjects); + + public static IList ReadAllRepositories(string repositoriesRoot, DependencyGraphSpecProvider provider) + { + var directories = new DirectoryInfo(repositoriesRoot).GetDirectories(); + var repositories = new Repository[directories.Length]; + + var sw = Stopwatch.StartNew(); + Parallel.For(0, directories.Length, new ParallelOptions { MaxDegreeOfParallelism = 6 }, i => + { + var directoryInfo = directories[i]; + Console.WriteLine($"Gathering dependency information from {directoryInfo.Name}."); + + var repository = Read(provider, directoryInfo.Name, directoryInfo.FullName); + repositories[i] = repository; + + Console.WriteLine($"Done gathering dependency information from {directoryInfo.Name}."); + }); + sw.Stop(); + + Console.WriteLine($"Done reading dependency information for all repos in {sw.Elapsed}."); + + return repositories; + } + + public bool Equals(Repository other) => string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); + + public override int GetHashCode() => StringComparer.OrdinalIgnoreCase.GetHashCode(Name); + + private static Repository Read(DependencyGraphSpecProvider provider, string name, string repositoryPath) + { + var repository = new Repository(name); + + ReadSharedSourceProjects(Path.Combine(repositoryPath, "shared"), repository, repository.Projects); + var srcDirectory = Path.Combine(repositoryPath, "src"); + + var solutionFiles = Directory.EnumerateFiles(repositoryPath, "*.sln"); + var knownProjects = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var file in solutionFiles) + { + var spec = provider.GetDependencyGraphSpec(name, file); + foreach (var specProject in spec.Projects) + { + if (!knownProjects.Add(specProject.FilePath) || + specProject.RestoreMetadata.ProjectStyle != ProjectStyle.PackageReference) + { + continue; + } + + var projectPath = Path.GetFullPath(specProject.FilePath); + + var project = new Project(specProject.Name) + { + PackageReferences = GetPackageReferences(specProject), + Repository = repository, + }; + + var projectGroup = projectPath.StartsWith(srcDirectory, StringComparison.OrdinalIgnoreCase) ? + repository.Projects : + repository.SupportProjects; + projectGroup.Add(project); + } + } + + return repository; + } + + private static List GetPackageReferences(NuGet.ProjectModel.PackageSpec specProject) + { + var allDependencies = Enumerable.Concat( + specProject.Dependencies, + specProject.TargetFrameworks.SelectMany(tfm => tfm.Dependencies)) + .Distinct(); + + var packageReferences = allDependencies + .Where(d => d.LibraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)) + .Select(d => d.Name) + .ToList(); + return packageReferences; + } + + private static void ReadSharedSourceProjects(string sharedSourceProjectsRoot, Repository repository, IList projects) + { + if (!Directory.Exists(sharedSourceProjectsRoot)) + { + return; + } + + foreach (var directory in new DirectoryInfo(sharedSourceProjectsRoot).EnumerateDirectories()) + { + var project = new Project(directory.Name) + { + Repository = repository, + }; + projects.Add(project); + } + } + } +} \ No newline at end of file diff --git a/tools/BuildGraph/TopologicalSort.cs b/tools/BuildGraph/TopologicalSort.cs new file mode 100644 index 000000000..308d9eef7 --- /dev/null +++ b/tools/BuildGraph/TopologicalSort.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BuildGraph +{ + public class TopologicalSort : IComparer + { + public static readonly TopologicalSort Instance = new TopologicalSort(); + + public int Compare(GraphNode x, GraphNode y) + { + var xScore = GetOrder(x); + var yScore = GetOrder(y); + return xScore.CompareTo(yScore); + } + + public static int GetOrder(GraphNode node) + { + var visited = new List(); + return GetOrder(node, visited); + } + + private static int GetOrder(GraphNode node, List visited) + { + if (visited.Contains(node)) + { + var cycle = string.Join(" -> ", visited.Select(v => v.Repository.Name)); + throw new Exception($"Cycle detected in the build graph: {cycle} -> {node.Repository.Name}."); + } + + var score = 0; + visited.Add(node); + foreach (var dependentNode in node.Incoming) + { + score = Math.Max(score, GetOrder(dependentNode, visited)); + } + visited.RemoveAt(visited.Count - 1); + + return score + 1; + } + } +} diff --git a/tools/PinVersion.sln b/tools/PinVersion.sln deleted file mode 100644 index 654e556bc..000000000 --- a/tools/PinVersion.sln +++ /dev/null @@ -1,22 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.23107.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "PinVersion", "PinVersion\PinVersion.xproj", "{3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/tools/PinVersion/PinVersion.xproj b/tools/PinVersion/PinVersion.xproj deleted file mode 100644 index 7e71282c5..000000000 --- a/tools/PinVersion/PinVersion.xproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - 14.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - - - - 3d3b5750-0c24-4f1c-9304-b5e4d85cf5a0 - PinVersion - ..\artifacts\obj\$(MSBuildProjectName) - .\bin\ - - - 2.0 - - - \ No newline at end of file diff --git a/tools/PinVersion/Program.cs b/tools/PinVersion/Program.cs deleted file mode 100644 index 840f96e35..000000000 --- a/tools/PinVersion/Program.cs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NuGet.Common; -using NuGet.Protocol.Core.Types; -using NuGet.Protocol.Core.v3; -using NuGet.Versioning; - -namespace PinVersion -{ - public class Program - { - private static ConcurrentDictionary> _packageVersionLookup = - new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); - - public static void Main(string[] args) - { - if (args.Length != 3) - { - Console.Error.WriteLine("Usage "); - } - - var packageSource = args[0]; - var korebuildTag = args[1]; - - var repositoryNames = File.ReadAllLines(args[2]); - - Task.WaitAll(repositoryNames - .Select(repositoryPath => ExecuteAsync(repositoryPath, packageSource, korebuildTag)) - .ToArray()); - } - - public static async Task ExecuteAsync(string repositoryPath, string packageSource, string korebuildTag) - { - var packageRepository = Repository.Factory.GetCoreV3(packageSource); - var metadataResource = await packageRepository.GetResourceAsync(); - - // Pin project.json files - foreach (var file in Directory.EnumerateFiles(repositoryPath, "project.json", SearchOption.AllDirectories)) - { - var projectJson = JObject.Parse(File.ReadAllText(file)); - var projectName = Path.GetFileName(Path.GetDirectoryName(file)); - var latestPackageVersion = await GetOrAddVersion(metadataResource, projectName); - if (latestPackageVersion != null) - { - ((JValue)projectJson["version"]).Value = latestPackageVersion.ToNormalizedString(); - } - - var frameworkDependencies = projectJson["frameworks"] - ?.Cast() - ?.Select(f => ((JObject)f.Value)["dependencies"]) - ?? Enumerable.Empty(); - var dependencies = Enumerable.Concat(new[] { projectJson["dependencies"] }, frameworkDependencies) - .Where(d => d != null) - .SelectMany(d => d) - .Cast(); - - foreach (var dependency in dependencies) - { - latestPackageVersion = await GetOrAddVersion(metadataResource, dependency.Name); - if (latestPackageVersion != null) - { - if (dependency.Value.Type == JTokenType.Object) - { - // "key": { "version": "1.0.0-*", "type": "build" } - var value = (JObject)dependency.Value; - value["version"] = latestPackageVersion.ToNormalizedString(); - } - else - { - // "key": "version" - dependency.Value = latestPackageVersion.ToNormalizedString(); - } - } - } - - using (var fileWriter = new JsonTextWriter(new StreamWriter(file))) - { - fileWriter.Formatting = Formatting.Indented; - fileWriter.Indentation = 2; - projectJson.WriteTo(fileWriter); - } - - // Update KoreBuild path - - var buildFiles = new[] { "build.ps1", "build.sh" }; - foreach (var buildFile in buildFiles) - { - var buildFilePath = Path.Combine(repositoryPath, buildFile); - if (File.Exists(buildFilePath)) - { - var content = File.ReadAllText(buildFilePath); - var replaced = content.Replace("KoreBuild/archive/release.zip", $"KoreBuild/archive/{korebuildTag}.zip"); - - if (content != replaced) - { - File.WriteAllText(buildFilePath, replaced); - } - } - } - } - } - - private static Task GetOrAddVersion(MetadataResource resource, string packageId) - { - return _packageVersionLookup.GetOrAdd(packageId, id => - { - return resource.GetLatestVersion( - packageId, - includePrerelease: true, - includeUnlisted: false, - log: NullLogger.Instance, - token: default(CancellationToken)); - }); - } - } -} diff --git a/tools/PinVersion/project.json b/tools/PinVersion/project.json deleted file mode 100644 index cad5f425f..000000000 --- a/tools/PinVersion/project.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "1.0.0-*", - "buildOptions": { - "emitEntryPoint": true - }, - "dependencies": { - "Newtonsoft.Json": "9.0.1", - "NuGet.Protocol.Core.v3": "3.5.0-beta-final" - }, - "frameworks": { - "net451": { } - } -} diff --git a/tools/PinVersions/PinVersionUtility.cs b/tools/PinVersions/PinVersionUtility.cs new file mode 100644 index 000000000..cd1559824 --- /dev/null +++ b/tools/PinVersions/PinVersionUtility.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using NuGet.Common; +using NuGet.Frameworks; +using NuGet.LibraryModel; +using NuGet.ProjectModel; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; +using UniverseTools; + +namespace PinVersions +{ + class PinVersionUtility + { + private readonly string _repositoryRoot; + private readonly FindPackageByIdResource[] _findPackageResources; + private readonly ConcurrentDictionary> _exactMatches = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); + private readonly DependencyGraphSpecProvider _provider; + private readonly SourceCacheContext _sourceCacheContext; + + public PinVersionUtility(string repositoryRoot, List pinSources, DependencyGraphSpecProvider provider) + { + _repositoryRoot = repositoryRoot; + _findPackageResources = new FindPackageByIdResource[pinSources.Count]; + for (var i = 0; i < pinSources.Count; i++) + { + var repository = FactoryExtensionsV3.GetCoreV3(Repository.Factory, pinSources[i].Trim()); + _findPackageResources[i] = repository.GetResource(); + } + _provider = provider; + _sourceCacheContext = new SourceCacheContext(); + } + + public void Execute() + { + var repositoryDirectoryInfo = new DirectoryInfo(_repositoryRoot); + var knownProjects = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var slnFile in repositoryDirectoryInfo.EnumerateFiles("*.sln")) + { + var graphSpec = _provider.GetDependencyGraphSpec(repositoryDirectoryInfo.Name, slnFile.FullName); + foreach (var specProject in graphSpec.Projects) + { + if (!knownProjects.Add(specProject.FilePath) || + specProject.RestoreMetadata.ProjectStyle != ProjectStyle.PackageReference) + { + continue; + } + + var projectFileInfo = new FileInfo(specProject.FilePath); + var pinnedReferencesFile = Path.Combine( + specProject.RestoreMetadata.OutputPath, + projectFileInfo.Name + ".pinnedversions.targets"); + + Directory.CreateDirectory(Path.GetDirectoryName(pinnedReferencesFile)); + + var allDependencies = specProject.Dependencies.Select(dependency => new { Dependency = dependency, FrameworkName = NuGetFramework.AnyFramework }) + .Concat(specProject.TargetFrameworks.SelectMany(tfm => tfm.Dependencies.Select(dependency => new { Dependency = dependency, tfm.FrameworkName }))) + .Where(d => d.Dependency.LibraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)); + + var packageReferencesItemGroup = new XElement("ItemGroup"); + foreach (var dependency in allDependencies) + { + var reference = dependency.Dependency; + var versionRange = reference.LibraryRange.VersionRange; + if (!versionRange.IsFloating) + { + continue; + } + + var exactVersion = GetExactVersion(reference.Name, versionRange); + if (exactVersion == null) + { + continue; + } + + var metadata = new List + { + new XAttribute("Update", reference.Name), + new XAttribute("Version", exactVersion.ToNormalizedString()), + }; + + if (dependency.FrameworkName != NuGetFramework.AnyFramework) + { + metadata.Add(new XAttribute("Condition", $"'$(TargetFramework)'=='{dependency.FrameworkName.GetShortFolderName()}'")); + } + + packageReferencesItemGroup.Add(new XElement("PackageReference", metadata)); + } + + var pinnedVersionRoot = new XElement("Project", packageReferencesItemGroup); + File.WriteAllText(pinnedReferencesFile, pinnedVersionRoot.ToString()); + } + } + } + + private NuGetVersion GetExactVersion(string name, VersionRange range) + { + if (range.MinVersion == null) + { + throw new Exception($"Unsupported version range {range}."); + } + + if (!_exactMatches.TryGetValue(name, out var versionTask)) + { + versionTask = _exactMatches.GetOrAdd(name, GetExactVersionAsync(name, range.MinVersion)); + } + + return versionTask.Result; + } + + private async Task GetExactVersionAsync(string name, NuGetVersion floatingVersion) + { + foreach (var findPackageResource in _findPackageResources) + { + var packageVersions = await findPackageResource.GetAllVersionsAsync(name, _sourceCacheContext, NullLogger.Instance, default(CancellationToken)); + + var matchingVersions = packageVersions.Where(v => v.Version == floatingVersion.Version).ToList(); + switch (matchingVersions.Count) + { + case 0: + return null; + case 1: + return matchingVersions[0]; + default: + throw new Exception($"More than one version for {name} found that matches the specified version constraint: {string.Join(" ", matchingVersions)}."); + } + } + + return null; + } + } +} diff --git a/tools/PinVersions/PinVersions.csproj b/tools/PinVersions/PinVersions.csproj new file mode 100644 index 000000000..441cf455d --- /dev/null +++ b/tools/PinVersions/PinVersions.csproj @@ -0,0 +1,16 @@ + + + + Exe + netcoreapp1.1 + + + + + + + + + + + \ No newline at end of file diff --git a/tools/PinVersions/Program.cs b/tools/PinVersions/Program.cs new file mode 100644 index 000000000..f197fecf5 --- /dev/null +++ b/tools/PinVersions/Program.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.CommandLineUtils; +using UniverseTools; + +namespace PinVersions +{ + class Program + { + static int Main(string[] args) + { + var app = new CommandLineApplication(); + + var pinSourceOption = app.Option("-s|--source", + "Feed containing packages to pin.", + CommandOptionType.MultipleValue); + + var packageSpecsDirectoryOption = app.Option("--graph-specs-root", + "Directory containing package specs. (Optional)", + CommandOptionType.SingleValue); + + var repositoryArgument = app.Argument("Repository", "Repository directory"); + + app.OnExecute(() => + { + if (!pinSourceOption.HasValue()) + { + Console.Error.WriteLine($"Option {pinSourceOption.Template} must have a value."); + return 1; + } + + if (string.IsNullOrEmpty(repositoryArgument.Value)) + { + Console.Error.WriteLine($"Repository argument must be specified."); + return 1; + } + + var graphSpecProvider = packageSpecsDirectoryOption.HasValue() ? + new DependencyGraphSpecProvider(packageSpecsDirectoryOption.Value().Trim()) : + DependencyGraphSpecProvider.Default; + + using (graphSpecProvider) + { + var pinVersionUtility = new PinVersionUtility(repositoryArgument.Value.Trim(), pinSourceOption.Values, graphSpecProvider); + pinVersionUtility.Execute(); + } + + return 0; + }); + + return app.Execute(args); + } + } +} \ No newline at end of file diff --git a/tools/shared/DependencyGraphSpecProvider.cs b/tools/shared/DependencyGraphSpecProvider.cs new file mode 100644 index 000000000..8a498b50c --- /dev/null +++ b/tools/shared/DependencyGraphSpecProvider.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using NuGet.ProjectModel; + +namespace UniverseTools +{ + public class DependencyGraphSpecProvider : IDisposable + { + private readonly string _packageSpecDirectory; + private readonly bool _deleteSpecDirectoryOnDispose; + + public DependencyGraphSpecProvider(string packageSpecDirectory) + : this(packageSpecDirectory, deleteSpecDirectoryOnDispose: false) + { + } + + private DependencyGraphSpecProvider(string packageSpecDirectory, bool deleteSpecDirectoryOnDispose) + { + _packageSpecDirectory = packageSpecDirectory; + _deleteSpecDirectoryOnDispose = deleteSpecDirectoryOnDispose; + } + + public static DependencyGraphSpecProvider Default { get; } = + new DependencyGraphSpecProvider(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), deleteSpecDirectoryOnDispose: true); + + public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath) + { + var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json"); + + if (!File.Exists(outputFile)) + { + RunMSBuild(solutionPath, outputFile); + } + + return DependencyGraphSpec.Load(outputFile); + } + + private static void RunMSBuild(string solutionPath, string outputFile) + { + var psi = new ProcessStartInfo(DotNetMuxer.MuxerPathOrDefault()); + + var arguments = new List + { + "msbuild", + $"\"{solutionPath}\"", + "/t:GenerateRestoreGraphFile", + "/nologo", + "/v:q", + "/p:BuildProjectReferences=false", + $"/p:RestoreGraphOutputPath=\"{outputFile}\"", + }; + + psi.Arguments = string.Join(" ", arguments); + psi.RedirectStandardOutput = true; + + var process = new Process + { + StartInfo = psi, + EnableRaisingEvents = true, + }; + process.OutputDataReceived += (sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + Console.WriteLine(args.Data); + } + }; + + using (process) + { + process.Start(); + process.BeginOutputReadLine(); + + process.WaitForExit(60 * 5000); + if (process.ExitCode != 0) + { + throw new Exception($"{psi.FileName} {psi.Arguments} failed. Exit code {process.ExitCode}."); + } + } + } + + public void Dispose() + { + if (_deleteSpecDirectoryOnDispose) + { + Directory.Delete(_packageSpecDirectory, recursive: true); + } + } + } +} diff --git a/tools/shared/DotNetMuxer.cs b/tools/shared/DotNetMuxer.cs new file mode 100644 index 000000000..5c61b12ae --- /dev/null +++ b/tools/shared/DotNetMuxer.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Runtime.InteropServices; + +// Copy of code from https://github.com/aspnet/DotNetTools/blob/047e4a8533bbcdf22831c40dd9d9aaf0c80d1953/shared/DotNetMuxer.cs +namespace UniverseTools +{ + public static class DotNetMuxer + { + private const string MuxerName = "dotnet"; + + static DotNetMuxer() + { + MuxerPath = TryFindMuxerPath(); + } + + public static string MuxerPath { get; } + + public static string MuxerPathOrDefault() + => MuxerPath ?? MuxerName; + + private static string TryFindMuxerPath() + { + var fileName = MuxerName; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + fileName += ".exe"; + } + + var fxDepsFile = AppContext.GetData("FX_DEPS_FILE") as string; + + if (string.IsNullOrEmpty(fxDepsFile)) + { + return null; + } + + var muxerDir = new FileInfo(fxDepsFile) // Microsoft.NETCore.App.deps.json + .Directory? // (version) + .Parent? // Microsoft.NETCore.App + .Parent? // shared + .Parent; // DOTNET_HOME + + if (muxerDir == null) + { + return null; + } + + var muxer = Path.Combine(muxerDir.FullName, fileName); + return File.Exists(muxer) + ? muxer + : null; + } + } +} \ No newline at end of file