[vcpkg] Implementation of --x-binarysource=nuget (and friends) (#12058)

* [vcpkg] Initial implementation of --x-binarysource=nuget * [vcpkg] Remove double-double quoting of CMake arguments * [vcpkg] Update nuget.exe to 5.5.1 to support Azure DevOps Artifacts * [vcpkg] Enable batching of NuGet server calls with prefetch(). Add `interactive` binarysource. * [vcpkg] Add `nugetconfig` binary source * [vcpkg] Short circuit querying remote NuGet servers once all refs are found * [vcpkg] Add experimental help for binary caching * [vcpkg] Improved NuGet cache package descriptions and version formatting * [vcpkg] Rename `CmdLineBuilder::build()` to extract() * [vcpkg-help] Ascii-betize help topics * [vcpkg] Add tests for cmdlinebuilder. Improve handling of quotes and slashes. * [vcpkg] Addressing code review comments * [vcpkg] Add tests for vcpkg::reformat_version() * [vcpkg] Added test for vcpkg::generate_nuspec() * [vcpkg] Add tests for vcpkg::XmlSerializer * [vcpkg] Addressed code review comment * [vcpkg] Add test for vcpkg::Strings::find_first_of * [vcpkg] Fix machine-specific paths in test for vcpkg::generate_nuspec() Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
microsoft · Jun 26, 2020 · 91e798a · 91e798a · gvanem · Jun 29, 2020
1 parent 7ebb42a
commit 91e798a
Show file tree

Hide file tree

Showing 19 changed files with 1,154 additions and 90 deletions.
diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml
@@ -60,10 +60,10 @@
         <sha512>f477842d0cebefcd6bf9c6d536ab8ea20ec5b0aa967ee963ab6a101aeff9df8742ca600d35f39e2e7158d76d8231f1ed2bef6104dce84d2bf8d6b07d17d706a1</sha512>
     </tool>
     <tool name="nuget" os="windows">
-        <version>4.8.1</version>
+        <version>5.5.1</version>
         <exeRelativePath>nuget.exe</exeRelativePath>
-        <url>https://dist.nuget.org/win-x86-commandline/v4.8.1/nuget.exe</url>
-        <sha512>42cb744338af8decc033a75bce5b4c4df28e102bafc45f9a8ba86d7bc010f5b43ebacae80d7b28c4f85ac900eefc2a349620ae65f27f6ca1c21c53b63b92924b</sha512>
+        <url>https://dist.nuget.org/win-x86-commandline/v5.5.1/nuget.exe</url>
+        <sha512>22ea847d8017cd977664d0b13c889cfb13c89143212899a511be217345a4e243d4d8d4099700114a11d26a087e83eb1a3e2b03bdb5e0db48f10403184cd26619</sha512>
     </tool>
     <tool name="installerbase" os="windows">
         <version>3.1.81</version>

diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h
@@ -175,6 +175,8 @@ namespace vcpkg::Strings
 
     std::vector<std::string> split(const std::string& s, const char delimiter);
 
+    const char* find_first_of(StringView searched, StringView candidates);
+
     std::vector<StringView> find_all_enclosed(StringView input, StringView left_delim, StringView right_delim);
 
     StringView find_exactly_one_enclosed(StringView input, StringView left_tag, StringView right_tag);

diff --git a/toolsrc/include/vcpkg/base/system.process.h b/toolsrc/include/vcpkg/base/system.process.h
@@ -23,6 +23,18 @@ namespace vcpkg::System
                                      const fs::path& cmake_script,
                                      const std::vector<CMakeVariable>& pass_variables);
 
+    struct CmdLineBuilder
+    {
+        CmdLineBuilder& path_arg(const fs::path& p) { return string_arg(p.u8string()); }
+        CmdLineBuilder& string_arg(StringView s);
+        std::string extract() noexcept { return std::move(buf); }
+
+        operator ZStringView() const { return buf; }
+
+    private:
+        std::string buf;
+    };
+
     fs::path get_exe_path_of_current_process();
 
     struct ExitCodeAndOutput

diff --git a/toolsrc/include/vcpkg/base/util.h b/toolsrc/include/vcpkg/base/util.h
@@ -49,6 +49,19 @@ namespace vcpkg::Util
         }
     }
 
+    template<class Range, class Pred, class E = ElementT<Range>>
+    std::vector<E> filter(const Range& xs, Pred&& f)
+    {
+        std::vector<E> ret;
+
+        for (auto&& x : xs)
+        {
+            if (f(x)) ret.push_back(x);
+        }
+
+        return ret;
+    }
+
     template<class Range, class Func>
     using FmapOut = std::remove_reference_t<decltype(std::declval<Func&>()(*std::declval<Range>().begin()))>;
 

diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h
@@ -8,6 +8,7 @@
 namespace vcpkg::Dependencies
 {
     struct InstallPlanAction;
+    struct ActionPlan;
 }
 namespace vcpkg::Build
 {
@@ -27,10 +28,17 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        virtual void prefetch() = 0;
+        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
+        /// `plan`.
+        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+        /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        /// Called upon a failure during the build of `action`
         virtual void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) = 0;
+        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
+        /// missing packages.
         virtual RestoreResult precheck(const VcpkgPaths& paths,
                                        const Dependencies::InstallPlanAction& action,
                                        bool purge_tombstones) = 0;
@@ -42,4 +50,6 @@ namespace vcpkg
                                                                                     View<std::string> args);
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs_pure(const std::string& env_string,
                                                                                          View<std::string> args);
+
+    void help_topic_binary_caching(const VcpkgPaths& paths);
 }
diff --git a/toolsrc/include/vcpkg/binarycaching.private.h b/toolsrc/include/vcpkg/binarycaching.private.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <string>
+#include <vcpkg/dependencies.h>
+#include <vcpkg/packagespec.h>
+#include <vcpkg/vcpkgpaths.h>
+
+namespace vcpkg
+{
+    std::string reformat_version(const std::string& version, const std::string& abi_tag);
+
+    struct NugetReference
+    {
+        explicit NugetReference(const Dependencies::InstallPlanAction& action)
+            : NugetReference(action.spec,
+                             action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)
+                                 .source_control_file->core_paragraph->version,
+                             action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi)
+        {
+        }
+
+        NugetReference(const PackageSpec& spec, const std::string& raw_version, const std::string& abi_tag)
+            : id(spec.dir()), version(reformat_version(raw_version, abi_tag))
+        {
+        }
+
+        std::string id;
+        std::string version;
+
+        std::string nupkg_filename() const { return Strings::concat(id, '.', version, ".nupkg"); }
+    };
+
+    std::string generate_nuspec(const VcpkgPaths& paths,
+                                const Dependencies::InstallPlanAction& action,
+                                const NugetReference& ref);
+
+    struct XmlSerializer
+    {
+        std::string buf;
+        int indent = 0;
+
+        XmlSerializer& emit_declaration();
+        XmlSerializer& open_tag(StringLiteral sl);
+        XmlSerializer& start_complex_open_tag(StringLiteral sl);
+        XmlSerializer& text_attr(StringLiteral name, StringView content);
+        XmlSerializer& finish_complex_open_tag();
+        XmlSerializer& finish_self_closing_complex_tag();
+        XmlSerializer& close_tag(StringLiteral sl);
+        XmlSerializer& text(StringView sv);
+        XmlSerializer& simple_tag(StringLiteral tag, StringView content);
+        XmlSerializer& line_break();
+    };
+
+}
diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h
@@ -284,7 +284,8 @@ namespace vcpkg::Build
     struct AbiInfo
     {
         std::unique_ptr<PreBuildInfo> pre_build_info;
-        const Toolset* toolset;
+        Optional<const Toolset&> toolset;
+        Optional<const std::string&> triplet_abi;
         std::string package_abi;
         Optional<fs::path> abi_tag_file;
     };

diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
@@ -92,6 +92,7 @@ namespace vcpkg
         void example(StringView example_text);
         void header(StringView name);
         void blank();
+        void text(StringView text, int indent = 0);
 
         std::string m_str;
     };

diff --git a/toolsrc/src/vcpkg-test/binarycaching.cpp b/toolsrc/src/vcpkg-test/binarycaching.cpp
@@ -0,0 +1,138 @@
+#include <catch2/catch.hpp>
+#include <vcpkg/binarycaching.private.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/vcpkgcmdarguments.h>
+#include <vcpkg/sourceparagraph.h>
+#include <vcpkg/paragraphs.h>
+#include <vcpkg/dependencies.h>
+#include <string>
+
+using namespace vcpkg;
+
+TEST_CASE ("reformat_version semver-ish", "[reformat_version]")
+{
+    REQUIRE(reformat_version("0.0.0", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("1.0.1", "abitag") == "1.0.1-abitag");
+    REQUIRE(reformat_version("1.01.000", "abitag") == "1.1.0-abitag");
+    REQUIRE(reformat_version("1.2", "abitag") == "1.2.0-abitag");
+    REQUIRE(reformat_version("v52", "abitag") == "52.0.0-abitag");
+    REQUIRE(reformat_version("v09.01.02", "abitag") == "9.1.2-abitag");
+    REQUIRE(reformat_version("1.1.1q", "abitag") == "1.1.1-abitag");
+    REQUIRE(reformat_version("1", "abitag") == "1.0.0-abitag");
+}
+
+TEST_CASE ("reformat_version date", "[reformat_version]")
+{
+    REQUIRE(reformat_version("2020-06-26", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("20-06-26", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("2020-06-26-release", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("2020-06-26000", "abitag") == "2020.6.26-abitag");
+}
+
+TEST_CASE ("reformat_version generic", "[reformat_version]")
+{
+    REQUIRE(reformat_version("apr", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("", "abitag") == "0.0.0-abitag");
+}
+
+TEST_CASE ("generate_nuspec", "[generate_nuspec]")
+{
+    auto& fsWrapper = Files::get_real_filesystem();
+    VcpkgCmdArguments args = VcpkgCmdArguments::create_from_arg_sequence(nullptr, nullptr);
+    args.packages_root_dir = std::make_unique<std::string>("/");
+    VcpkgPaths paths(fsWrapper, args);
+
+    auto pghs = Paragraphs::parse_paragraphs(R"(
+Source: zlib2
+Version: 1.5
+Build-Depends: zlib
+Description: a spiffy compression library wrapper
+
+Feature: a
+Description: a feature
+
+Feature: b
+Description: enable bzip capabilities
+Build-Depends: bzip
+)",
+                                             "<testdata>");
+    REQUIRE(pghs.has_value());
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    REQUIRE(maybe_scf.has_value());
+    SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
+
+    Dependencies::InstallPlanAction ipa(PackageSpec{"zlib2", Triplet::X64_WINDOWS},
+                                        scfl,
+                                        Dependencies::RequestType::USER_REQUESTED,
+                                        {{"a", {}}, {"b", {}}});
+
+    ipa.abi_info = Build::AbiInfo{};
+    ipa.abi_info.get()->package_abi = "packageabi";
+    std::string tripletabi("tripletabi");
+    ipa.abi_info.get()->triplet_abi = tripletabi;
+
+    NugetReference ref(ipa);
+
+    REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-packageabi.nupkg");
+
+    auto nuspec = generate_nuspec(paths, ipa, ref);
+#ifdef _WIN32
+#define PKGPATH "C:\\zlib2_x64-windows\\**"
+#else
+#define PKGPATH "/zlib2_x64-windows/**"
+#endif
+    std::string expected = R"(<package>
+  <metadata>
+    <id>zlib2_x64-windows</id>
+    <version>1.5.0-packageabi</version>
+    <authors>vcpkg</authors>
+    <description>NOT FOR DIRECT USE. Automatically generated cache package.
+
+a spiffy compression library wrapper
+
+Version: 1.5
+Triplet/Compiler hash: tripletabi
+Features: a, b
+Dependencies:
+</description>
+    <packageTypes><packageType name="vcpkg"/></packageTypes>
+    </metadata>
+  <files><file src=")" PKGPATH R"(" target=""/></files>
+  </package>
+)";
+    auto expected_lines = Strings::split(expected, '\n');
+    auto nuspec_lines = Strings::split(nuspec, '\n');
+    for (size_t i = 0; i < expected_lines.size() && i < nuspec_lines.size(); ++i)
+    {
+        INFO("on line: " << i);
+        REQUIRE(nuspec_lines[i] == expected_lines[i]);
+    }
+    REQUIRE(nuspec_lines.size() == expected_lines.size());
+}
+
+TEST_CASE ("XmlSerializer", "[XmlSerializer]")
+{
+    XmlSerializer xml;
+    xml.open_tag("a");
+    xml.open_tag("b");
+    xml.simple_tag("c", "d");
+    xml.close_tag("b");
+    xml.text("escaping: & < > \" '");
+
+    REQUIRE(xml.buf == R"(<a><b><c>d</c></b>escaping: &amp; &lt; &gt; &quot; &apos;)");
+
+    xml = XmlSerializer();
+    xml.emit_declaration();
+    xml.start_complex_open_tag("a").text_attr("b", "<").text_attr("c", "  ").finish_self_closing_complex_tag();
+    REQUIRE(xml.buf == R"(<?xml version="1.0" encoding="utf-8"?><a b="&lt;" c="  "/>)");
+
+    xml = XmlSerializer();
+    xml.start_complex_open_tag("a").finish_complex_open_tag();
+    REQUIRE(xml.buf == R"(<a>)");
+
+    xml = XmlSerializer();
+    xml.line_break();
+    xml.open_tag("a").line_break().line_break();
+    xml.close_tag("a").line_break().line_break();
+    REQUIRE(xml.buf == "\n<a>\n  \n  </a>\n\n");
+}