diff --git a/azure-pipelines/e2e_assets/fetch/ninja.bat b/azure-pipelines/e2e_assets/fetch/ninja.bat new file mode 100644 index 0000000000..30dc2947b2 --- /dev/null +++ b/azure-pipelines/e2e_assets/fetch/ninja.bat @@ -0,0 +1 @@ +@echo 0.0.0 diff --git a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 index b6a97c1739..257508003e 100644 --- a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 @@ -23,6 +23,13 @@ if (-not $IsMacOS -and -not $IsLinux) { 6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3 ninja-win-1.10.2.zip + + 1.10.2 + ninja.exe + https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip + 6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3 + ninja-win-1.10.2.zip + 3.22.2 cmake-3.22.2-windows-i386\bin\cmake.exe @@ -62,4 +69,30 @@ if (-not $IsMacOS -and -not $IsLinux) { Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot")) Throw-IfFailed Require-FileExists "$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows/ninja.exe" + + $path = $env:PATH + + $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows" + Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) + Throw-IfFailed + Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + + $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1" + Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) + Throw-IfFailed + Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + + Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue + Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES + + $env:VCPKG_FORCE_SYSTEM_BINARIES = "1" + $env:PATH = "$PSScriptRoot\..\e2e_assets\fetch;$path" + Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) + Throw-IfFailed + Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + + Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES + Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) + Throw-IfFailed + Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" } diff --git a/azure-pipelines/end-to-end-tests.ps1 b/azure-pipelines/end-to-end-tests.ps1 index 06e9f095d1..1cc8a9dc74 100755 --- a/azure-pipelines/end-to-end-tests.ps1 +++ b/azure-pipelines/end-to-end-tests.ps1 @@ -74,6 +74,8 @@ $n = 1 $m = $AllTests.Count $envvars_clear = @( + "VCPKG_FORCE_SYSTEM_BINARIES", + "VCPKG_FORCE_DOWNLOADED_BINARIES", "VCPKG_DEFAULT_HOST_TRIPLET", "VCPKG_DEFAULT_TRIPLET", "VCPKG_BINARY_SOURCES", @@ -84,7 +86,7 @@ $envvars_clear = @( "VCPKG_FEATURE_FLAGS", "VCPKG_DISABLE_METRICS" ) -$envvars = $envvars_clear + @("VCPKG_DOWNLOADS", "X_VCPKG_REGISTRIES_CACHE") +$envvars = $envvars_clear + @("VCPKG_DOWNLOADS", "X_VCPKG_REGISTRIES_CACHE", "PATH") foreach ($Test in $AllTests) { diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 8148632fa1..8c5d9797d3 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -63,6 +63,12 @@ namespace Catch { static const std::string convert(const vcpkg::PackageSpec& value) { return value.to_string(); } }; + + template<> + struct StringMaker + { + static const std::string convert(const vcpkg::Path& value) { return "\"" + value.native() + "\""; } + }; } namespace vcpkg diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index 18f0d7e233..ae5fdbaca2 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -7,6 +7,8 @@ #include +#include + namespace vcpkg { // Extract `archive` to `to_path` using `tar_tool`. @@ -14,20 +16,20 @@ namespace vcpkg // Extract `archive` to `to_path` using `cmake_tool`. (CMake's built in tar) void extract_tar_cmake(const Path& cmake_tool, const Path& archive, const Path& to_path); // Extract `archive` to `to_path`, deleting `to_path` first. - void extract_archive(const VcpkgPaths& paths, const Path& archive, const Path& to_path); + void extract_archive(Filesystem& fs, const ToolCache& tools, const Path& archive, const Path& to_path); #ifdef _WIN32 // Extract the 7z archive part of a self extracting 7z installer - void win32_extract_self_extracting_7z(const VcpkgPaths& paths, const Path& archive, const Path& to_path); + void win32_extract_self_extracting_7z(Filesystem& fs, const Path& archive, const Path& to_path); // Extract `archive` to `to_path`, deleting `to_path` first. `archive` must be a zip file. // This function will use potentially less performant tools that are reliably available on any machine. - void win32_extract_bootstrap_zip(const VcpkgPaths& paths, const Path& archive, const Path& to_path); + void win32_extract_bootstrap_zip(Filesystem& fs, const ToolCache& tools, const Path& archive, const Path& to_path); #endif // Compress the source directory into the destination file. - int compress_directory_to_zip(const VcpkgPaths& paths, const Path& source, const Path& destination); + int compress_directory_to_zip(Filesystem& fs, const ToolCache& tools, const Path& source, const Path& destination); - Command decompress_zip_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path); + Command decompress_zip_archive_cmd(const ToolCache& tools, const Path& dst, const Path& archive_path); std::vector decompress_in_parallel(View jobs); } diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 85261cd295..2fdb9c9db0 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -1,6 +1,8 @@ #pragma once -#include +#include +#include + #include #include #include diff --git a/include/vcpkg/base/fwd/downloads.h b/include/vcpkg/base/fwd/downloads.h new file mode 100644 index 0000000000..7b1890b8f6 --- /dev/null +++ b/include/vcpkg/base/fwd/downloads.h @@ -0,0 +1,7 @@ +#pragma once + +namespace vcpkg +{ + struct DownloadManager; + struct DownloadManagerConfig; +} diff --git a/include/vcpkg/base/messages.h b/include/vcpkg/base/messages.h index e7d122083f..258551a21e 100644 --- a/include/vcpkg/base/messages.h +++ b/include/vcpkg/base/messages.h @@ -279,6 +279,7 @@ namespace vcpkg::msg DECLARE_MSG_ARG(vendor, "Azure"); DECLARE_MSG_ARG(version, "1.3.8"); DECLARE_MSG_ARG(action_index, "340"); + DECLARE_MSG_ARG(env_var, "VCPKG_DEFAULT_TRIPLET"); #undef DECLARE_MSG_ARG // These are `...` instead of diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h index c68cb3ff73..4e24a7e223 100644 --- a/include/vcpkg/tools.h +++ b/include/vcpkg/tools.h @@ -1,11 +1,11 @@ #pragma once +#include +#include #include #include -#include -#include #include #include @@ -31,8 +31,6 @@ namespace vcpkg static constexpr StringLiteral ARIA2 = "aria2"; static constexpr StringLiteral NODE = "node"; static constexpr StringLiteral IFW_INSTALLER_BASE = "ifw_installerbase"; - static constexpr StringLiteral IFW_BINARYCREATOR = "ifw_binarycreator"; - static constexpr StringLiteral IFW_REPOGEN = "ifw_repogen"; // This duplicate of 7zip uses msiexec to unpack, which is a fallback for Windows 7. static constexpr StringLiteral SEVEN_ZIP_MSI = "7zip_msi"; } @@ -41,12 +39,16 @@ namespace vcpkg { virtual ~ToolCache() = default; - virtual const Path& get_tool_path_from_system(const Filesystem& fs, StringView tool) const = 0; - virtual const Path& get_tool_path(const VcpkgPaths& paths, StringView tool) const = 0; - virtual const std::string& get_tool_version(const VcpkgPaths& paths, StringView tool) const = 0; + virtual const Path& get_tool_path(StringView tool) const = 0; + virtual const std::string& get_tool_version(StringView tool) const = 0; }; - Optional> parse_tool_version_string(StringView string_version); + ExpectedL find_system_tar(const Filesystem& fs); - std::unique_ptr get_tool_cache(RequireExactVersions abiToolVersionHandling); + std::unique_ptr get_tool_cache(Filesystem& fs, + std::shared_ptr downloader, + Path downloads, + Path xml_config, + Path tools, + RequireExactVersions abiToolVersionHandling); } diff --git a/include/vcpkg/tools.test.h b/include/vcpkg/tools.test.h new file mode 100644 index 0000000000..13bc749875 --- /dev/null +++ b/include/vcpkg/tools.test.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include + +namespace vcpkg +{ + struct ToolData + { + std::string name; + + std::array version; + // relative path inside tool_dir_subpath + Path exe_subpath; + std::string url; + // relative path from paths.downloads + Path download_subpath; + bool is_archive; + // relative path from paths.tools + Path tool_dir_subpath; + std::string sha512; + + Path exe_path(const Path& tools_base_path) const { return tools_base_path / tool_dir_subpath / exe_subpath; } + }; + + Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool, StringView os); + + Optional> parse_tool_version_string(StringView string_version); +} diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index 6a164c1ae0..c3100b265f 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -1,11 +1,13 @@ #pragma once +#include #include #include #include #include #include +#include #include #include @@ -40,8 +42,6 @@ namespace vcpkg std::vector supported_architectures; }; - struct DownloadManager; - namespace Build { struct PreBuildInfo; @@ -114,20 +114,21 @@ namespace vcpkg const std::unique_ptr m_pimpl; public: + const Path& scripts; + const Path& downloads; + const Path& tools; const Path builtin_registry_versions; - const Path scripts; const Path prefab; const Path buildsystems; const Path buildsystems_msbuild_targets; const Path buildsystems_msbuild_props; - const Path downloads; - const Path tools; const Path ports_cmake; const Path triplets; const Path community_triplets; std::string get_toolver_diagnostics() const; + const ToolCache& get_tool_cache() const; const Path& get_tool_exe(StringView tool) const; const std::string& get_tool_version(StringView tool) const; diff --git a/locales/messages.en.json b/locales/messages.en.json index d9363fe0b4..560f140254 100644 --- a/locales/messages.en.json +++ b/locales/messages.en.json @@ -66,9 +66,10 @@ "CiBaselineRegressionHeader": "REGRESSIONS:", "CiBaselineUnexpectedPass": "PASSING, REMOVE FROM FAIL LIST: {spec} ({path}).", "CmakeTargetsExcluded": "note: {count} additional targets are not displayed.", - "CommandFailed": "command:\n{command_line}\nfailed with the following results:\n{list}\n", + "CommandFailed": "command:\n{command_line}\nfailed with the following results:\n", "CouldNotDeduceNugetIdAndVersion": "Could not deduce nuget id and version from filename: {path}", "CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===\n", + "DownloadAvailable": "A downloadable copy of this tool is available and can be used by unsetting {env_var}.", "DownloadedSources": "Downloaded sources for {spec}", "DownloadingVcpkgCeBundle": "Downloading vcpkg-ce bundle {version}...", "DownloadingVcpkgCeBundleLatest": "Downloading latest vcpkg-ce bundle...", @@ -111,6 +112,9 @@ "HeaderOnlyUsage": "{package_name} is header-only and can be used from CMake via:", "IllegalFeatures": "error: List of features is not allowed in this context", "IllegalPlatformSpec": "error: Platform qualifier is not allowed in this context", + "InstallWithSystemManager": "You may be able to install this tool via your system package manager.", + "InstallWithSystemManagerMono": "Ubuntu 18.04 users may need a newer version of mono, available at {url}.", + "InstallWithSystemManagerPkg": "You may be able to install this tool via your system package manager ({command_line}).", "InstallingPackage": "Installing {action_index}/{count} {spec}...", "InternalErrorMessage": "internal error: ", "InternalErrorMessageContact": "Please open an issue at https://github.com/microsoft/vcpkg/issues/new?template=other-type-of-bug-report.md&labels=category:vcpkg-bug with detailed steps to reproduce the problem.", @@ -159,9 +163,12 @@ "ResultsHeader": "RESULTS", "SeeURL": "See {url} for more information.", "SourceFieldPortNameMismatch": "The 'Source' field inside the CONTROL file, or \"name\" field inside the vcpkg.json file has the name {package_name} and does not match the port directory {path}.", + "ToolFetchFailed": "Could not fetch {tool_name}.", + "ToolInWin10": "This utility is bundled with Windows 10 or later.", "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.", "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'", + "UnknownTool": "vcpkg does not have a definition of this tool for this platform.", "UnsupportedSystemName": "Error: Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", "UnsupportedToolchain": "Error: in triplet {triplet}: Unable to find a valid toolchain combination.\n The requested target architecture was {arch}\n The selected Visual Studio instance is at {path}\n The available toolchain combinations are {list}\n", "UpdateBaselineAddBaselineNoManifest": "the --{option} switch was passed, but there is no manifest file to add a `builtin-baseline` field to.", diff --git a/locales/messages.json b/locales/messages.json index c5cd6d6be6..c160e24fcf 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -121,10 +121,14 @@ "_CiBaselineUnexpectedPass.comment": "example of {spec} is 'zlib:x64-windows'.\nexample of {path} is '/foo/bar'.\n", "CmakeTargetsExcluded": "note: {count} additional targets are not displayed.", "_CmakeTargetsExcluded.comment": "example of {count} is '42'.\n", + "CommandFailed": "command:\n{command_line}\nfailed with the following results:\n", + "_CommandFailed.comment": "example of {command_line} is 'vcpkg install zlib'.\n", "CouldNotDeduceNugetIdAndVersion": "Could not deduce nuget id and version from filename: {path}", "_CouldNotDeduceNugetIdAndVersion.comment": "example of {path} is '/foo/bar'.\n", "CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===\n", "_CurlReportedUnexpectedResults.comment": "{command_line} is the command line to call curl.exe, {actual} is the console output of curl.exe locale-invariant download results.\nexample of {command_line} is 'vcpkg install zlib'.\n", + "DownloadAvailable": "A downloadable copy of this tool is available and can be used by unsetting {env_var}.", + "_DownloadAvailable.comment": "example of {env_var} is 'VCPKG_DEFAULT_TRIPLET'.\n", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "example of {spec} is 'zlib:x64-windows'.\n", "DownloadingVcpkgCeBundle": "Downloading vcpkg-ce bundle {version}...", @@ -191,6 +195,11 @@ "_HeaderOnlyUsage.comment": "'header' refers to C/C++ .h files\nexample of {package_name} is 'zlib'.\n", "IllegalFeatures": "error: List of features is not allowed in this context", "IllegalPlatformSpec": "error: Platform qualifier is not allowed in this context", + "InstallWithSystemManager": "You may be able to install this tool via your system package manager.", + "InstallWithSystemManagerMono": "Ubuntu 18.04 users may need a newer version of mono, available at {url}.", + "_InstallWithSystemManagerMono.comment": "example of {url} is 'https://github.com/microsoft/vcpkg'.\n", + "InstallWithSystemManagerPkg": "You may be able to install this tool via your system package manager ({command_line}).", + "_InstallWithSystemManagerPkg.comment": "example of {command_line} is 'vcpkg install zlib'.\n", "InstallingPackage": "Installing {action_index}/{count} {spec}...", "_InstallingPackage.comment": "example of {action_index} is '340'.\nexample of {count} is '42'.\nexample of {spec} is 'zlib:x64-windows'.\n", "InternalErrorMessage": "internal error: ", @@ -273,9 +282,13 @@ "_SeeURL.comment": "example of {url} is 'https://github.com/microsoft/vcpkg'.\n", "SourceFieldPortNameMismatch": "The 'Source' field inside the CONTROL file, or \"name\" field inside the vcpkg.json file has the name {package_name} and does not match the port directory {path}.", "_SourceFieldPortNameMismatch.comment": "{package_name} and {path} are both names of installable ports/packages. 'Source', 'CONTROL', 'vcpkg.json', and 'name' references are locale-invariant.\nexample of {package_name} is 'zlib'.\nexample of {path} is '/foo/bar'.\n", + "ToolFetchFailed": "Could not fetch {tool_name}.", + "_ToolFetchFailed.comment": "example of {tool_name} is 'aria2'.\n", + "ToolInWin10": "This utility is bundled with Windows 10 or later.", "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.", "UnknownBaselineFileContent": "unrecognizable baseline entry; expected 'port:triplet=(fail|skip)'", "UnknownBinaryProviderType": "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', 'interactive', and 'files'", + "UnknownTool": "vcpkg does not have a definition of this tool for this platform.", "UnsupportedSystemName": "Error: Could not map VCPKG_CMAKE_SYSTEM_NAME '{system_name}' to a vcvarsall platform. Supported system names are '', 'Windows' and 'WindowsStore'.", "_UnsupportedSystemName.comment": "example of {system_name} is 'Darwin'.\n", "UnsupportedToolchain": "Error: in triplet {triplet}: Unable to find a valid toolchain combination.\n The requested target architecture was {arch}\n The selected Visual Studio instance is at {path}\n The available toolchain combinations are {list}\n", diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 1b046ac4bd..0ecb2b4861 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -1,6 +1,7 @@ #include #include +#include using namespace vcpkg; diff --git a/src/vcpkg-test/tools.cpp b/src/vcpkg-test/tools.cpp index 61bb380623..82d4c4ef9b 100644 --- a/src/vcpkg-test/tools.cpp +++ b/src/vcpkg-test/tools.cpp @@ -1,9 +1,14 @@ #include +#include + #include +#include #include +#include + using namespace vcpkg; TEST_CASE ("parse_tool_version_string", "[tools]") @@ -41,3 +46,78 @@ Copyright (C) 2006, 2019 Tatsuhiro Tsujikawa)"); result = parse_tool_version_string("4"); CHECK_FALSE(result.has_value()); } + +TEST_CASE ("parse_tool_data_from_xml", "[tools]") +{ + const StringView tool_doc = R"( + + + + 2.7.4 + + + + + + 5.11.0 + nuget.exe + https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe + 06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda83b64c03ce14c8f35ca9957a17a8c02b8c4b7 + + + 16.12.0 + node-v16.12.0-win-x64\node.exe + https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z + 0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023fb7068ed1f8a7678e446260c3db3537fa888 + node-v16.12.0-win-x64.7z + + +)"; + + { + auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "tool1", "windows"); + REQUIRE(!data.has_value()); + } + { + auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "node", "unknown"); + REQUIRE(!data.has_value()); + } + { + auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "node", "windows"); + REQUIRE(data.has_value()); + auto& p = *data.get(); + CHECK(p.is_archive); + CHECK(p.version == decltype(p.version){16, 12, 0}); + CHECK(p.tool_dir_subpath == "node-16.12.0-windows"); + CHECK(p.exe_subpath == "node-v16.12.0-win-x64\\node.exe"); + CHECK(p.download_subpath == "node-v16.12.0-win-x64.7z"); + CHECK(p.sha512 == "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" + "fb7068ed1f8a7678e446260c3db3537fa888"); + CHECK(p.url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); + } + { + auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "nuget", "osx"); + REQUIRE(data.has_value()); + auto& p = *data.get(); + CHECK_FALSE(p.is_archive); + CHECK(p.version == decltype(p.version){5, 11, 0}); + CHECK(p.tool_dir_subpath == "nuget-5.11.0-osx"); + CHECK(p.exe_subpath == "nuget.exe"); + CHECK(p.download_subpath == "06a337c9-nuget.exe"); + CHECK(p.sha512 == "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda8" + "3b64c03ce14c8f35ca9957a17a8c02b8c4b7"); + CHECK(p.url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); + } + { + auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "git", "linux"); + REQUIRE(data.has_value()); + auto& p = *data.get(); + CHECK_FALSE(p.is_archive); + CHECK(p.version == decltype(p.version){2, 7, 4}); + CHECK(p.tool_dir_subpath == "git-2.7.4-linux"); + CHECK(p.exe_subpath == ""); + CHECK(p.download_subpath == ""); + CHECK(p.sha512 == ""); + CHECK(p.url == ""); + } +} \ No newline at end of file diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index bb706d5b1a..b6eea54226 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -1,11 +1,12 @@ +#include #include #include #include +#include #include #include #include -#include namespace { @@ -22,9 +23,9 @@ namespace "Could not deduce nuget id and version from filename: {path}"); #if defined(_WIN32) - void win32_extract_nupkg(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + void win32_extract_nupkg(const ToolCache& tools, const Path& archive, const Path& to_path) { - const auto nuget_exe = paths.get_tool_exe(Tools::NUGET); + const auto nuget_exe = tools.get_tool_path(Tools::NUGET); const auto stem = archive.stem(); @@ -49,7 +50,7 @@ namespace .string_arg("-OutputDirectory") .string_arg(to_path) .string_arg("-Source") - .string_arg(paths.downloads) + .string_arg(archive.parent_path()) .string_arg("-nocache") .string_arg("-DirectDownload") .string_arg("-NonInteractive") @@ -132,13 +133,13 @@ namespace } #endif // ^^^ _WIN32 - void extract_archive_to_empty(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + void extract_archive_to_empty(Filesystem& fs, const ToolCache& tools, const Path& archive, const Path& to_path) { const auto ext = archive.extension(); #if defined(_WIN32) if (Strings::case_insensitive_ascii_equals(ext, ".nupkg")) { - win32_extract_nupkg(paths, archive, to_path); + win32_extract_nupkg(tools, archive, to_path); } else if (Strings::case_insensitive_ascii_equals(ext, ".msi")) { @@ -147,17 +148,18 @@ namespace else if (Strings::case_insensitive_ascii_equals(ext, ".zip") || Strings::case_insensitive_ascii_equals(ext, ".7z")) { - extract_tar_cmake(paths.get_tool_exe(Tools::CMAKE), archive, to_path); + extract_tar_cmake(tools.get_tool_path(Tools::CMAKE), archive, to_path); } else if (Strings::case_insensitive_ascii_equals(ext, ".exe")) { const Path filename = archive.filename(); const Path stem = filename.stem(); const Path to_archive = Path(archive.parent_path()) / stem; - win32_extract_self_extracting_7z(paths, archive, to_archive); - extract_archive_to_empty(paths, to_archive, to_path); + win32_extract_self_extracting_7z(fs, archive, to_archive); + extract_archive_to_empty(fs, tools, to_archive, to_path); } #else + (void)fs; if (ext == ".zip") { const auto code = @@ -167,7 +169,7 @@ namespace #endif else if (ext == ".gz" || ext == ".bz2" || ext == ".tgz") { - vcpkg::extract_tar(paths.get_tool_exe(Tools::TAR), archive, to_path); + vcpkg::extract_tar(tools.get_tool_path(Tools::TAR), archive, to_path); } else { @@ -175,9 +177,11 @@ namespace } } - Path extract_archive_to_temp_subdirectory(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + Path extract_archive_to_temp_subdirectory(Filesystem& fs, + const ToolCache& tools, + const Path& archive, + const Path& to_path) { - Filesystem& fs = paths.get_filesystem(); Path to_path_partial = to_path + ".partial"; #if defined(_WIN32) to_path_partial += "." + std::to_string(GetCurrentProcessId()); @@ -185,7 +189,7 @@ namespace fs.remove_all(to_path_partial, VCPKG_LINE_INFO); fs.create_directories(to_path_partial, VCPKG_LINE_INFO); - extract_archive_to_empty(paths, archive, to_path_partial); + extract_archive_to_empty(fs, tools, archive, to_path_partial); return to_path_partial; } } @@ -193,7 +197,7 @@ namespace namespace vcpkg { #ifdef _WIN32 - void win32_extract_self_extracting_7z(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + void win32_extract_self_extracting_7z(Filesystem& fs, const Path& archive, const Path& to_path) { constexpr static const char header_7z[] = "7z\xBC\xAF\x27\x1C"; @@ -204,7 +208,6 @@ namespace vcpkg "Unable to extract 7z archive from Installer %s. Missing '.7z.exe' extension.", archive); - Filesystem& fs = paths.get_filesystem(); auto contents = fs.read_contents(archive, VCPKG_LINE_INFO); const auto pos = contents.find(header_7z); Checks::check_exit(VCPKG_LINE_INFO, @@ -215,9 +218,8 @@ namespace vcpkg fs.write_contents(to_path, std::move(contents), VCPKG_LINE_INFO); } - void win32_extract_bootstrap_zip(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + void win32_extract_bootstrap_zip(Filesystem& fs, const ToolCache& tools, const Path& archive, const Path& to_path) { - Filesystem& fs = paths.get_filesystem(); fs.remove_all(to_path, VCPKG_LINE_INFO); Path to_path_partial = to_path + ".partial." + std::to_string(GetCurrentProcessId()); @@ -238,7 +240,7 @@ namespace vcpkg // Example: // msiexec unpacks 7zip_msi unpacks cmake unpacks 7zip unpacks git - win32_extract_with_seven_zip(paths.get_tool_exe(Tools::SEVEN_ZIP_MSI), archive, to_path_partial); + win32_extract_with_seven_zip(tools.get_tool_path(Tools::SEVEN_ZIP_MSI), archive, to_path_partial); } fs.rename_with_retry(to_path_partial, to_path, VCPKG_LINE_INFO); } @@ -260,20 +262,18 @@ namespace vcpkg Checks::check_exit(VCPKG_LINE_INFO, code == 0, "tar failed while extracting %s", archive); } - void extract_archive(const VcpkgPaths& paths, const Path& archive, const Path& to_path) + void extract_archive(Filesystem& fs, const ToolCache& tools, const Path& archive, const Path& to_path) { - Filesystem& fs = paths.get_filesystem(); fs.remove_all(to_path, VCPKG_LINE_INFO); - Path to_path_partial = extract_archive_to_temp_subdirectory(paths, archive, to_path); + Path to_path_partial = extract_archive_to_temp_subdirectory(fs, tools, archive, to_path); fs.rename_with_retry(to_path_partial, to_path, VCPKG_LINE_INFO); } - int compress_directory_to_zip(const VcpkgPaths& paths, const Path& source, const Path& destination) + int compress_directory_to_zip(Filesystem& fs, const ToolCache& tools, const Path& source, const Path& destination) { - auto& fs = paths.get_filesystem(); fs.remove(destination, VCPKG_LINE_INFO); #if defined(_WIN32) - auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); + auto&& seven_zip_exe = tools.get_tool_path(Tools::SEVEN_ZIP); return cmd_execute_and_capture_output( Command{seven_zip_exe}.string_arg("a").string_arg(destination).string_arg(source / "*"), @@ -282,6 +282,7 @@ namespace vcpkg .exit_code; #else + (void)tools; return cmd_execute_clean(Command{"zip"} .string_arg("--quiet") .string_arg("-y") @@ -294,18 +295,18 @@ namespace vcpkg #endif } - Command decompress_zip_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path) + Command decompress_zip_archive_cmd(const ToolCache& tools, const Path& dst, const Path& archive_path) { Command cmd; #if defined(_WIN32) - auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); + auto&& seven_zip_exe = tools.get_tool_path(Tools::SEVEN_ZIP); cmd.string_arg(seven_zip_exe) .string_arg("x") .string_arg(archive_path) .string_arg("-o" + dst.native()) .string_arg("-y"); #else - (void)paths; + (void)tools; cmd.string_arg("unzip").string_arg("-qq").string_arg(archive_path).string_arg("-d" + dst.native()); #endif return cmd; diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 9ade74acf2..39db0196da 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -331,7 +331,7 @@ namespace { auto pkg_path = paths.package_dir(spec); clean_prepare_dir(fs, pkg_path); - jobs.push_back(decompress_zip_archive_cmd(paths, pkg_path, archive_path)); + jobs.push_back(decompress_zip_archive_cmd(paths.get_tool_cache(), pkg_path, archive_path)); action_idxs.push_back(i); archive_paths.push_back(std::move(archive_path)); } @@ -393,7 +393,7 @@ namespace auto& fs = paths.get_filesystem(); const auto archive_subpath = make_archive_subpath(abi_tag); const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec); - int code = compress_directory_to_zip(paths, paths.package_dir(spec), tmp_archive_path); + int code = compress_directory_to_zip(fs, paths.get_tool_cache(), paths.package_dir(spec), tmp_archive_path); if (code != 0) { vcpkg::print2( @@ -547,8 +547,9 @@ namespace if (codes[i] == 200) { action_idxs.push_back(i); - jobs.push_back(decompress_zip_archive_cmd( - paths, paths.package_dir(actions[url_indices[i]].spec), url_paths[i].second)); + jobs.push_back(decompress_zip_archive_cmd(paths.get_tool_cache(), + paths.package_dir(actions[url_indices[i]].spec), + url_paths[i].second)); } } auto job_results = decompress_in_parallel(jobs); @@ -1040,7 +1041,8 @@ namespace auto&& action = actions[url_indices[idx]]; auto&& url_path = url_paths[idx]; if (!download_file(url_path.first, url_path.second)) continue; - jobs.push_back(decompress_zip_archive_cmd(paths, paths.package_dir(action.spec), url_path.second)); + jobs.push_back(decompress_zip_archive_cmd( + paths.get_tool_cache(), paths.package_dir(action.spec), url_path.second)); idxs.push_back(idx); } @@ -1081,7 +1083,8 @@ namespace const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); auto& spec = action.spec; const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec); - int code = compress_directory_to_zip(paths, paths.package_dir(spec), tmp_archive_path); + int code = compress_directory_to_zip( + paths.get_filesystem(), paths.get_tool_cache(), paths.package_dir(spec), tmp_archive_path); if (code != 0) { vcpkg::print2( diff --git a/src/vcpkg/build.cpp b/src/vcpkg/build.cpp index 84d3f313dc..58bf359061 100644 --- a/src/vcpkg/build.cpp +++ b/src/vcpkg/build.cpp @@ -759,11 +759,9 @@ namespace vcpkg::Build {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, }); - if (!get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - const Path& git_exe_path = paths.get_tool_exe(Tools::GIT); - out_vars.push_back({"GIT", git_exe_path}); - } + // Make sure GIT could be found + const Path& git_exe_path = paths.get_tool_exe(Tools::GIT); + out_vars.push_back({"GIT", git_exe_path}); } static CompilerInfo load_compiler_info(const VcpkgPaths& paths, const AbiInfo& abi_info) @@ -1262,6 +1260,7 @@ namespace vcpkg::Build abi_tag_entries.emplace_back("cmake", paths.get_tool_version(Tools::CMAKE)); + // This #ifdef is mirrored in tools.cpp's PowershellProvider #if defined(_WIN32) abi_tag_entries.emplace_back("powershell", paths.get_tool_version("powershell-core")); #endif diff --git a/src/vcpkg/cmakevars.cpp b/src/vcpkg/cmakevars.cpp index 4e23d89a6c..bcba5305a6 100644 --- a/src/vcpkg/cmakevars.cpp +++ b/src/vcpkg/cmakevars.cpp @@ -249,12 +249,11 @@ endfunction() } DECLARE_AND_REGISTER_MESSAGE(CommandFailed, - (msg::command_line, msg::list), + (msg::command_line), "", "command:\n" "{command_line}\n" - "failed with the following results:\n" - "{list}\n"); + "failed with the following results:\n"); void TripletCMakeVarProvider::launch_and_split( const Path& script_path, std::vector>>& vars) const @@ -274,11 +273,11 @@ endfunction() if (exit_code != 0) { - msg::println(Color::error, - msgCommandFailed, - msg::command_line = cmd_launch_cmake.command_line(), - msg::list = Strings::join(", ", lines)); - Checks::exit_fail(VCPKG_LINE_INFO); + Checks::msg_exit_with_message( + VCPKG_LINE_INFO, + msg::format(msgCommandFailed, msg::command_line = cmd_launch_cmake.command_line()) + .appendnl() + .append_raw(Strings::join(", ", lines))); } const auto end = lines.cend(); diff --git a/src/vcpkg/commands.zbootstrap-standalone.cpp b/src/vcpkg/commands.zbootstrap-standalone.cpp index d237143ee1..fe3a67362a 100644 --- a/src/vcpkg/commands.zbootstrap-standalone.cpp +++ b/src/vcpkg/commands.zbootstrap-standalone.cpp @@ -46,9 +46,7 @@ namespace vcpkg::Commands download_manager.download_file(fs, bundle_uri, bundle_tarball, nullopt); #endif // ^^^ !VCPKG_STANDALONE_BUNDLE_SHA - auto tool_cache = get_tool_cache(RequireExactVersions::NO); - const auto tar = tool_cache->get_tool_path_from_system(fs, Tools::TAR); - extract_tar(tar, bundle_tarball, vcpkg_root); + extract_tar(find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), bundle_tarball, vcpkg_root); fs.remove(bundle_tarball, VCPKG_LINE_INFO); Checks::exit_success(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/export.ifw.cpp b/src/vcpkg/export.ifw.cpp index b69cd17a7c..9ad7409aa2 100644 --- a/src/vcpkg/export.ifw.cpp +++ b/src/vcpkg/export.ifw.cpp @@ -354,7 +354,8 @@ namespace vcpkg::Export::IFW void do_repository(const std::string& export_id, const Options& ifw_options, const VcpkgPaths& paths) { - const Path& repogen_exe = paths.get_tool_exe(Tools::IFW_REPOGEN); + Path repogen_exe = paths.get_tool_exe(Tools::IFW_INSTALLER_BASE); + repogen_exe.replace_filename("repogen.exe"); const auto packages_dir = get_packages_dir_path(export_id, ifw_options, paths); const auto repository_dir = get_repository_dir_path(export_id, ifw_options, paths); @@ -375,7 +376,8 @@ namespace vcpkg::Export::IFW void do_installer(const std::string& export_id, const Options& ifw_options, const VcpkgPaths& paths) { - const Path& binarycreator_exe = paths.get_tool_exe(Tools::IFW_BINARYCREATOR); + Path binarycreator_exe = paths.get_tool_exe(Tools::IFW_INSTALLER_BASE); + binarycreator_exe.replace_filename("binarycreator.exe"); const auto config_file = get_config_file_path(export_id, ifw_options, paths); const auto packages_dir = get_packages_dir_path(export_id, ifw_options, paths); const auto repository_dir = get_repository_dir_path(export_id, ifw_options, paths); diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index ea6ae8c3ea..85bf093dac 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -636,9 +636,11 @@ namespace vcpkg::Export::Prefab "[DEBUG] Exporting AAR And POM\n\tAAR Path %s\n\tPOM Path %s\n", exported_archive_path, pom_path)); } - Checks::check_exit(VCPKG_LINE_INFO, - compress_directory_to_zip(paths, package_directory, exported_archive_path) != 0, - Strings::concat("Failed to compress folder ", package_directory)); + Checks::check_exit( + VCPKG_LINE_INFO, + compress_directory_to_zip( + paths.get_filesystem(), paths.get_tool_cache(), package_directory, exported_archive_path) != 0, + Strings::concat("Failed to compress folder ", package_directory)); std::string POM = R"( #include #include #include +#include #include +#include #include #include +#include #include #include #include #include #include -#include +#include +#include #include namespace vcpkg { - struct ToolData - { - std::array version; - Path exe_path; - std::string url; - Path download_path; - bool is_archive; - Path tool_dir_path; - std::string sha512; - }; + DECLARE_AND_REGISTER_MESSAGE(ToolFetchFailed, (msg::tool_name), "", "Could not fetch {tool_name}."); + DECLARE_AND_REGISTER_MESSAGE(ToolInWin10, (), "", "This utility is bundled with Windows 10 or later."); + DECLARE_AND_REGISTER_MESSAGE( + DownloadAvailable, + (msg::env_var), + "", + "A downloadable copy of this tool is available and can be used by unsetting {env_var}."); + DECLARE_AND_REGISTER_MESSAGE(UnknownTool, + (), + "", + "vcpkg does not have a definition of this tool for this platform."); // /\d+\.\d+(\.\d+)?/ Optional> parse_tool_version_string(StringView string_version) @@ -67,29 +73,29 @@ namespace vcpkg return std::array{*d1.get(), *d2.get(), *d3.get()}; } - static ExpectedT parse_tool_data_from_xml(const VcpkgPaths& paths, StringView tool) + static Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool) { #if defined(_WIN32) - static constexpr StringLiteral OS_STRING = "windows"; + return parse_tool_data_from_xml(XML, XML_PATH, tool, "windows"); #elif defined(__APPLE__) - static constexpr StringLiteral OS_STRING = "osx"; + return parse_tool_data_from_xml(XML, XML_PATH, tool, "osx"); #elif defined(__linux__) - static constexpr StringLiteral OS_STRING = "linux"; + return parse_tool_data_from_xml(XML, XML_PATH, tool, "linux"); #elif defined(__FreeBSD__) - static constexpr StringLiteral OS_STRING = "freebsd"; + return parse_tool_data_from_xml(XML, XML_PATH, tool, "freebsd"); #elif defined(__OpenBSD__) - static constexpr StringLiteral OS_STRING = "openbsd"; + return parse_tool_data_from_xml(XML, XML_PATH, tool, "openbsd"); #else - return std::string("operating system is unknown"); + return nullopt; #endif + } -#if defined(_WIN32) || defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) + Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool, StringView os) + { static const std::string XML_VERSION = "2"; - static const auto XML_PATH = paths.scripts / "vcpkgTools.xml"; static const std::regex XML_VERSION_REGEX{R"###()###"}; - static const std::string XML = paths.get_filesystem().read_contents(XML_PATH, VCPKG_LINE_INFO); - std::smatch match_xml_version; - const bool has_xml_version = std::regex_search(XML.cbegin(), XML.cend(), match_xml_version, XML_VERSION_REGEX); + std::cmatch match_xml_version; + const bool has_xml_version = std::regex_search(XML.begin(), XML.end(), match_xml_version, XML_VERSION_REGEX); Checks::check_exit( VCPKG_LINE_INFO, has_xml_version, R"(Could not find in %s)", XML_VERSION, XML_PATH); Checks::check_exit(VCPKG_LINE_INFO, @@ -99,29 +105,10 @@ namespace vcpkg XML_VERSION, match_xml_version[1]); - const std::regex tool_regex{Strings::format(R"###()###", tool, OS_STRING)}; - std::smatch match_tool_entry; - const bool has_tool_entry = std::regex_search(XML.cbegin(), XML.cend(), match_tool_entry, tool_regex); - if (!has_tool_entry) - { - StringLiteral add_info = ""; - if (tool == "mono") - { -#if defined(__APPLE__) - add_info = " (brew install mono)"; -#else - add_info = " (e.g. sudo apt install mono-complete). Ubuntu 18.04 users may " - "need a newer version of mono, available at https://www.mono-project.com/download/stable/"; -#endif - } - return Strings::format("Could not automatically acquire %s because there is no entry in %s for os=%s. You " - "may be able to install %s via your system package manager%s.", - tool, - XML_PATH, - OS_STRING, - tool, - add_info); - } + const std::regex tool_regex{Strings::format(R"###()###", tool, os)}; + std::cmatch match_tool_entry; + const bool has_tool_entry = std::regex_search(XML.begin(), XML.end(), match_tool_entry, tool_regex); + if (!has_tool_entry) return nullopt; const std::string tool_data = Strings::find_exactly_one_enclosed(XML, match_tool_entry[0].str(), "").to_string(); @@ -140,21 +127,25 @@ namespace vcpkg tool, version_as_string); - const std::string tool_dir_name = Strings::format("%s-%s-%s", tool, version_as_string, OS_STRING); - const auto tool_dir_path = paths.tools / tool_dir_name; - const auto exe_path = tool_dir_path / exe_relative_path; - Path download_path; + Path tool_dir_name = Strings::format("%s-%s-%s", tool, version_as_string, os); + Path download_subpath; if (auto a = archive_name.get()) { - download_path = paths.downloads / a->to_string(); + download_subpath = a->to_string(); } - else + else if (!exe_relative_path.empty()) { - download_path = paths.downloads / Strings::concat(sha512.substr(0, 8), '-', exe_relative_path); + download_subpath = Strings::concat(sha512.substr(0, 8), '-', exe_relative_path); } - return ToolData{*version.get(), exe_path, url, download_path, archive_name.has_value(), tool_dir_path, sha512}; -#endif + return ToolData{tool.to_string(), + *version.get(), + exe_relative_path, + url, + download_subpath, + archive_name.has_value(), + tool_dir_name, + sha512}; } struct PathAndVersion @@ -163,194 +154,73 @@ namespace vcpkg std::string version; }; - struct ToolProvider - { - virtual StringLiteral tool_data_name() const = 0; - virtual StringLiteral exe_stem() const = 0; - virtual std::array default_min_version() const = 0; + DECLARE_AND_REGISTER_MESSAGE(InstallWithSystemManager, + (), + "", + "You may be able to install this tool via your system package manager."); - virtual void add_special_paths(std::vector& out_candidate_paths) const { (void)out_candidate_paths; } - virtual ExpectedS get_version(const VcpkgPaths& paths, const Path& exe_path) const = 0; - }; + DECLARE_AND_REGISTER_MESSAGE( + InstallWithSystemManagerPkg, + (msg::command_line), + "", + "You may be able to install this tool via your system package manager ({command_line})."); - template - static Optional find_first_with_sufficient_version(const VcpkgPaths& paths, - const ToolProvider& tool_provider, - const std::vector& candidates, - Func&& accept_version) + struct ToolProvider { - const auto& fs = paths.get_filesystem(); - for (auto&& candidate : candidates) - { - if (!fs.exists(candidate, IgnoreErrors{})) continue; - auto maybe_version = tool_provider.get_version(paths, candidate); - const auto version = maybe_version.get(); - if (!version) continue; - const auto parsed_version = parse_tool_version_string(*version); - if (!parsed_version) continue; - auto& actual_version = *parsed_version.get(); - if (!accept_version(actual_version)) continue; - - return PathAndVersion{candidate, *version}; - } - - return nullopt; - } + virtual StringView tool_data_name() const = 0; + /// \returns The stem of the executable to search PATH for, or empty string if tool can't be searched + virtual StringView system_exe_stem() const = 0; + virtual std::array default_min_version() const = 0; + /// \returns \c true if the tool's version is included in package ABI calculations. ABI sensitive tools will be + /// pinned to exact versions if \c --x-abi-tools-use-exact-versions is passed. + virtual bool is_abi_sensitive() const = 0; - static Path fetch_tool(const VcpkgPaths& paths, StringView tool_name, const ToolData& tool_data) - { - const std::array& version = tool_data.version; - const std::string version_as_string = Strings::format("%d.%d.%d", version[0], version[1], version[2]); - Checks::check_maybe_upgrade(VCPKG_LINE_INFO, - !tool_data.url.empty(), - "A suitable version of %s was not found (required v%s) and unable to automatically " - "download a portable one. Please install a newer version of %s.", - tool_name, - version_as_string, - tool_name); - vcpkg::printf("A suitable version of %s was not found (required v%s). Downloading portable %s v%s...\n", - tool_name, - version_as_string, - tool_name, - version_as_string); - auto& fs = paths.get_filesystem(); - if (!fs.exists(tool_data.download_path, IgnoreErrors{})) - { - print2("Downloading ", tool_name, "...\n"); - print2(" ", tool_data.url, " -> ", tool_data.download_path, "\n"); - paths.get_download_manager().download_file(fs, tool_data.url, tool_data.download_path, tool_data.sha512); - } - else - { - verify_downloaded_file_hash(fs, tool_data.url, tool_data.download_path, tool_data.sha512); - } + virtual void add_system_paths(std::vector& out_candidate_paths) const { (void)out_candidate_paths; } + virtual ExpectedS get_version(const ToolCache& cache, const Path& exe_path) const = 0; - if (tool_data.is_archive) + virtual void add_system_package_info(LocalizedString& out) const { - print2("Extracting ", tool_name, "...\n"); -#if defined(_WIN32) - if (tool_name == "cmake") - { - // We use cmake as the core extractor on Windows, so we need to perform a special dance when - // extracting it. - win32_extract_bootstrap_zip(paths, tool_data.download_path, tool_data.tool_dir_path); - } - else -#endif // ^^^ _WIN32 - { - extract_archive(paths, tool_data.download_path, tool_data.tool_dir_path); - } + out.append_raw(" ").append(msgInstallWithSystemManager); } - else - { - fs.create_directories(tool_data.exe_path.parent_path(), IgnoreErrors{}); - fs.rename(tool_data.download_path, tool_data.exe_path, IgnoreErrors{}); - } - - Checks::check_exit(VCPKG_LINE_INFO, - fs.exists(tool_data.exe_path, IgnoreErrors{}), - "Expected %s to exist after fetching", - tool_data.exe_path); - - return tool_data.exe_path; - } - static PathAndVersion fetch_tool(const VcpkgPaths& paths, - const ToolProvider& tool_provider, - const ToolData& tool_data) - { - auto downloaded_path = fetch_tool(paths, tool_provider.tool_data_name(), tool_data); - auto downloaded_version = tool_provider.get_version(paths, downloaded_path).value_or_exit(VCPKG_LINE_INFO); - return {std::move(downloaded_path), std::move(downloaded_version)}; - } + bool is_system_searchable() const { return !system_exe_stem().empty(); } + }; - static bool should_force_vcpkg_downloaded_binaries(const ToolData* tool_data) - { - // If the tool doesn't have a downloaded URL, we cannot force the downloaded version - if (tool_data->url.empty()) - { - return false; - } - // If the VCPKG_FORCE_DOWNLOADED_BINARIES env var is set, it takes precedence - if (get_environment_variable("VCPKG_FORCE_DOWNLOADED_BINARIES").has_value()) - { - return true; - } - // Otherwise don't force downloaded binaries - return false; - } - static PathAndVersion get_path(const VcpkgPaths& paths, const ToolProvider& tool, bool exact_version = false) + struct GenericToolProvider : ToolProvider { - auto& fs = paths.get_filesystem(); + explicit GenericToolProvider(StringView tool) : m_tool_data_name(tool) { } - std::array min_version = tool.default_min_version(); + const StringView m_tool_data_name; - std::vector candidate_paths; - auto search_system_tool = true; // Search the tool in the system by default - auto maybe_tool_data = parse_tool_data_from_xml(paths, tool.tool_data_name()); - if (auto tool_data = maybe_tool_data.get()) - { - candidate_paths.push_back(tool_data->exe_path); - min_version = tool_data->version; - search_system_tool = !should_force_vcpkg_downloaded_binaries(tool_data); - } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return m_tool_data_name; } + virtual StringView system_exe_stem() const override { return {}; } + virtual std::array default_min_version() const override { return {0}; } - if (search_system_tool) + virtual ExpectedS get_version(const ToolCache&, const Path&) const override { - StringLiteral exe_stem = tool.exe_stem(); - if (!exe_stem.empty()) - { - auto paths_from_path = fs.find_from_PATH(exe_stem); - candidate_paths.insert(candidate_paths.end(), paths_from_path.cbegin(), paths_from_path.cend()); - } - - tool.add_special_paths(candidate_paths); + return {"0", expected_left_tag}; } - - const auto maybe_path = find_first_with_sufficient_version( - paths, tool, candidate_paths, [&min_version, exact_version](const std::array& actual_version) { - if (exact_version) - { - return actual_version[0] == min_version[0] && actual_version[1] == min_version[1] && - actual_version[2] == min_version[2]; - } - return actual_version[0] > min_version[0] || - (actual_version[0] == min_version[0] && actual_version[1] > min_version[1]) || - (actual_version[0] == min_version[0] && actual_version[1] == min_version[1] && - actual_version[2] >= min_version[2]); - }); - if (const auto p = maybe_path.get()) - { - return *p; - } - if (auto tool_data = maybe_tool_data.get()) - { - return fetch_tool(paths, tool, *tool_data); - } - - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO, maybe_tool_data.error()); - } + }; struct CMakeProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::CMAKE; } - virtual StringLiteral exe_stem() const override { return Tools::CMAKE; } + virtual bool is_abi_sensitive() const override { return true; } + virtual StringView tool_data_name() const override { return Tools::CMAKE; } + virtual StringView system_exe_stem() const override { return Tools::CMAKE; } virtual std::array default_min_version() const override { return {3, 17, 1}; } - virtual void add_special_paths(std::vector& out_candidate_paths) const override - { #if defined(_WIN32) + virtual void add_system_paths(std::vector& out_candidate_paths) const override + { const auto& program_files = get_program_files_platform_bitness(); if (const auto pf = program_files.get()) out_candidate_paths.push_back(*pf / "CMake" / "bin" / "cmake.exe"); const auto& program_files_32_bit = get_program_files_32_bit(); if (const auto pf = program_files_32_bit.get()) out_candidate_paths.push_back(*pf / "CMake" / "bin" / "cmake.exe"); -#else - // TODO: figure out if this should do anything on non-Windows - (void)out_candidate_paths; -#endif } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override +#endif + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -375,11 +245,12 @@ CMake suite maintained and supported by Kitware (kitware.com/cmake). struct NinjaProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::NINJA; } - virtual StringLiteral exe_stem() const override { return Tools::NINJA; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::NINJA; } + virtual StringView system_exe_stem() const override { return Tools::NINJA; } virtual std::array default_min_version() const override { return {3, 5, 1}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -398,17 +269,18 @@ CMake suite maintained and supported by Kitware (kitware.com/cmake). struct NuGetProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::NUGET; } - virtual StringLiteral exe_stem() const override { return Tools::NUGET; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::NUGET; } + virtual StringView system_exe_stem() const override { return Tools::NUGET; } virtual std::array default_min_version() const override { return {4, 6, 2}; } - virtual ExpectedS get_version(const VcpkgPaths& paths, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache& cache, const Path& exe_path) const override { Command cmd; #ifndef _WIN32 - cmd.string_arg(paths.get_tool_exe(Tools::MONO)); + cmd.string_arg(cache.get_tool_path(Tools::MONO)); #else - (void)paths; + (void)cache; #endif cmd.string_arg(exe_path); auto rc = cmd_execute_and_capture_output(cmd); @@ -443,10 +315,11 @@ Type 'NuGet help ' for help on a specific command. struct Aria2Provider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return "aria2"; } - virtual StringLiteral exe_stem() const override { return "aria2c"; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return "aria2"; } + virtual StringView system_exe_stem() const override { return "aria2c"; } virtual std::array default_min_version() const override { return {1, 33, 1}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -473,24 +346,22 @@ Copyright (C) 2006, 2019 Tatsuhiro Tsujikawa struct NodeProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::NODE; } - virtual StringLiteral exe_stem() const override { return Tools::NODE; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::NODE; } + virtual StringView system_exe_stem() const override { return Tools::NODE; } virtual std::array default_min_version() const override { return {16, 12, 0}; } - virtual void add_special_paths(std::vector& out_candidate_paths) const override - { #if defined(_WIN32) + virtual void add_system_paths(std::vector& out_candidate_paths) const override + { const auto& program_files = get_program_files_platform_bitness(); if (const auto pf = program_files.get()) out_candidate_paths.push_back(*pf / "nodejs" / "node.exe"); const auto& program_files_32_bit = get_program_files_32_bit(); if (const auto pf = program_files_32_bit.get()) out_candidate_paths.push_back(*pf / "nodejs" / "node.exe"); -#else - // TODO: figure out if this should do anything on non-windows - (void)out_candidate_paths; -#endif } +#endif - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -517,25 +388,23 @@ Copyright (C) 2006, 2019 Tatsuhiro Tsujikawa struct GitProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::GIT; } - virtual StringLiteral exe_stem() const override { return Tools::GIT; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::GIT; } + virtual StringView system_exe_stem() const override { return Tools::GIT; } virtual std::array default_min_version() const override { return {2, 7, 4}; } - virtual void add_special_paths(std::vector& out_candidate_paths) const override - { #if defined(_WIN32) + virtual void add_system_paths(std::vector& out_candidate_paths) const override + { const auto& program_files = get_program_files_platform_bitness(); if (const auto pf = program_files.get()) out_candidate_paths.push_back(*pf / "git" / "cmd" / "git.exe"); const auto& program_files_32_bit = get_program_files_32_bit(); if (const auto pf = program_files_32_bit.get()) out_candidate_paths.push_back(*pf / "git" / "cmd" / "git.exe"); -#else - // TODO: figure out if this should do anything on non-windows - (void)out_candidate_paths; -#endif } +#endif - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -555,13 +424,19 @@ git version 2.17.1.windows.2 } }; + DECLARE_AND_REGISTER_MESSAGE(InstallWithSystemManagerMono, + (msg::url), + "", + "Ubuntu 18.04 users may need a newer version of mono, available at {url}."); + struct MonoProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::MONO; } - virtual StringLiteral exe_stem() const override { return Tools::MONO; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::MONO; } + virtual StringView system_exe_stem() const override { return Tools::MONO; } virtual std::array default_min_version() const override { return {0, 0, 0}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto rc = cmd_execute_and_capture_output(Command(exe_path).string_arg("--version")); if (rc.exit_code != 0) @@ -578,15 +453,28 @@ Mono JIT compiler version 6.8.0.105 (Debian 6.8.0.105+dfsg-2 Wed Feb 26 23:23:50 VCPKG_LINE_INFO, idx != std::string::npos, "Unexpected format of mono version string: %s", rc.output); return {rc.output.substr(idx), expected_left_tag}; } + + virtual void add_system_package_info(LocalizedString& out) const override + { +#if defined(__APPLE__) + out.append_raw(" ").append(msgInstallWithSystemManagerPkg, msg::command_line = "brew install mono"); +#else + out.append_raw(" ").append(msgInstallWithSystemManagerPkg, + msg::command_line = "sudo apt install mono-complete"); + out.append_raw(" ").append(msgInstallWithSystemManagerMono, + msg::url = "https://www.mono-project.com/download/stable/"); +#endif + } }; struct GsutilProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::GSUTIL; } - virtual StringLiteral exe_stem() const override { return Tools::GSUTIL; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::GSUTIL; } + virtual StringView system_exe_stem() const override { return Tools::GSUTIL; } virtual std::array default_min_version() const override { return {4, 56, 0}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -609,11 +497,12 @@ gsutil version: 4.58 struct AwsCliProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::AWSCLI; } - virtual StringLiteral exe_stem() const override { return Tools::AWSCLI; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::AWSCLI; } + virtual StringView system_exe_stem() const override { return Tools::AWSCLI; } virtual std::array default_min_version() const override { return {2, 4, 4}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -636,11 +525,12 @@ aws-cli/2.4.4 Python/3.8.8 Windows/10 exe/AMD64 prompt/off struct CosCliProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return Tools::COSCLI; } - virtual StringLiteral exe_stem() const override { return Tools::COSCLI; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return Tools::COSCLI; } + virtual StringView system_exe_stem() const override { return "cos"; } virtual std::array default_min_version() const override { return {0, 11, 0}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -663,23 +553,12 @@ coscli version v0.11.0-beta struct IfwInstallerBaseProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return "installerbase"; } - virtual StringLiteral exe_stem() const override { return ""; } + virtual bool is_abi_sensitive() const override { return false; } + virtual StringView tool_data_name() const override { return "installerbase"; } + virtual StringView system_exe_stem() const override { return {}; } virtual std::array default_min_version() const override { return {0, 0, 0}; } - virtual void add_special_paths(std::vector& out_candidate_paths) const override - { - (void)out_candidate_paths; - // TODO: Uncomment later - // const std::vector from_path = find_from_PATH("installerbase"); - // candidate_paths.insert(candidate_paths.end(), from_path.cbegin(), from_path.cend()); - // candidate_paths.push_back(path(get_environment_variable("HOMEDRIVE").value_or("C:")) / - // "Qt" / "Tools" / "QtInstallerFramework" / "3.1" / "bin" / "installerbase.exe"); - // candidate_paths.push_back(path(get_environment_variable("HOMEDRIVE").value_or("C:")) / - // "Qt" / "QtIFW-3.1.0" / "bin" / "installerbase.exe"); - } - - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto cmd = Command(exe_path).string_arg("--framework-version"); auto rc = cmd_execute_and_capture_output(cmd); @@ -698,11 +577,20 @@ coscli version v0.11.0-beta struct PowerShellCoreProvider : ToolProvider { - virtual StringLiteral tool_data_name() const override { return "powershell-core"; } - virtual StringLiteral exe_stem() const override { return "pwsh"; } + virtual bool is_abi_sensitive() const override + { + // This #ifdef is mirrored in build.cpp's compute_abi_tag +#if defined(_WIN32) + return true; +#else + return false; +#endif + } + virtual StringView tool_data_name() const override { return "powershell-core"; } + virtual StringView system_exe_stem() const override { return "pwsh"; } virtual std::array default_min_version() const override { return {7, 0, 3}; } - virtual ExpectedS get_version(const VcpkgPaths&, const Path& exe_path) const override + virtual ExpectedS get_version(const ToolCache&, const Path& exe_path) const override { auto rc = cmd_execute_and_capture_output(Command(exe_path).string_arg("--version")); if (rc.exit_code != 0) @@ -726,171 +614,292 @@ coscli version v0.11.0-beta struct ToolCacheImpl final : ToolCache { - RequireExactVersions abiToolVersionHandling; - vcpkg::Cache system_cache; - vcpkg::Cache path_only_cache; + Filesystem& fs; + const std::shared_ptr downloader; + const Path downloads; + const Path xml_config; + const Path tools; + const RequireExactVersions abiToolVersionHandling; + + vcpkg::Lazy xml_config_contents; vcpkg::Cache path_version_cache; - ToolCacheImpl(RequireExactVersions abiToolVersionHandling) : abiToolVersionHandling(abiToolVersionHandling) { } - - virtual const Path& get_tool_path_from_system(const Filesystem& fs, StringView tool) const override + ToolCacheImpl(Filesystem& fs, + const std::shared_ptr& downloader, + Path downloads, + Path xml_config, + Path tools, + RequireExactVersions abiToolVersionHandling) + : fs(fs) + , downloader(downloader) + , downloads(std::move(downloads)) + , xml_config(std::move(xml_config)) + , tools(std::move(tools)) + , abiToolVersionHandling(abiToolVersionHandling) { - return system_cache.get_lazy(tool, [&] { - if (tool == Tools::TAR) - { - auto tars = fs.find_from_PATH("tar"); - if (tars.empty()) - { - Checks::exit_with_message(VCPKG_LINE_INFO, -#if defined(_WIN32) - "Could not find tar; the action you tried to take assumes Windows 10 " - "or later which are bundled with tar.exe." -#else // ^^^ _WIN32 // !_WIN32 vvv - "Could not find tar; please install it from your system package " - "manager." -#endif // ^^^ !_WIN32 + } - ); - } + template + Optional find_first_with_sufficient_version(const ToolProvider& tool_provider, + const std::vector& candidates, + Func&& accept_version) const + { + for (auto&& candidate : candidates) + { + if (!fs.exists(candidate, IgnoreErrors{})) continue; + auto maybe_version = tool_provider.get_version(*this, candidate); + const auto version = maybe_version.get(); + if (!version) continue; + const auto parsed_version = parse_tool_version_string(*version); + if (!parsed_version) continue; + auto& actual_version = *parsed_version.get(); + if (!accept_version(actual_version)) continue; + + return PathAndVersion{candidate, *version}; + } - return std::move(tars[0]); - } + return nullopt; + } + + Path download_tool(const ToolData& tool_data) const + { + const std::array& version = tool_data.version; + const std::string version_as_string = Strings::format("%d.%d.%d", version[0], version[1], version[2]); + Checks::check_maybe_upgrade( + VCPKG_LINE_INFO, + !tool_data.url.empty(), + "A suitable version of %s was not found (required v%s) and unable to automatically " + "download a portable one. Please install a newer version of %s.", + tool_data.name, + version_as_string, + tool_data.name); + vcpkg::printf("A suitable version of %s was not found (required v%s). Downloading portable %s v%s...\n", + tool_data.name, + version_as_string, + tool_data.name, + version_as_string); + const auto download_path = downloads / tool_data.download_subpath; + if (!fs.exists(download_path, IgnoreErrors{})) + { + print2("Downloading ", tool_data.name, "...\n"); + print2(" ", tool_data.url, " -> ", download_path, "\n"); + downloader->download_file(fs, tool_data.url, download_path, tool_data.sha512); + } + else + { + verify_downloaded_file_hash(fs, tool_data.url, download_path, tool_data.sha512); + } - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO, "Unknown or unavailable tool: %s", tool); - }); - } + const auto tool_dir_path = tools / tool_data.tool_dir_subpath; + Path exe_path = tool_dir_path / tool_data.exe_subpath; - virtual const Path& get_tool_path(const VcpkgPaths& paths, StringView tool) const override - { - return path_only_cache.get_lazy(tool, [&]() { - if (tool == Tools::IFW_BINARYCREATOR) + if (tool_data.is_archive) + { + print2("Extracting ", tool_data.name, "...\n"); +#if defined(_WIN32) + if (tool_data.name == "cmake") { - auto installer_base = get_tool_path(paths, Tools::IFW_INSTALLER_BASE); - installer_base.replace_filename("binarycreator.exe"); - return installer_base; + // We use cmake as the core extractor on Windows, so we need to perform a special dance when + // extracting it. + win32_extract_bootstrap_zip(fs, *this, download_path, tool_dir_path); } - - if (tool == Tools::IFW_REPOGEN) + else +#endif // ^^^ _WIN32 { - auto installer_base = get_tool_path(paths, Tools::IFW_INSTALLER_BASE); - installer_base.replace_filename("repogen.exe"); - return installer_base; + extract_archive(fs, *this, download_path, tool_dir_path); } + } + else + { + fs.create_directories(exe_path.parent_path(), IgnoreErrors{}); + fs.rename(download_path, exe_path, IgnoreErrors{}); + } - return get_tool_pathversion(paths, tool).p; - }); + Checks::check_exit( + VCPKG_LINE_INFO, fs.exists(exe_path, IgnoreErrors{}), "Expected %s to exist after fetching", exe_path); + + return exe_path; } - const PathAndVersion& get_tool_pathversion(const VcpkgPaths& paths, StringView tool) const + const std::string& get_config_contents() const { - return path_version_cache.get_lazy(tool, [&]() -> PathAndVersion { - // First deal with specially handled tools. - // For these we may look in locations like Program Files, the PATH etc as well as the auto-downloaded - // location. - if (tool == Tools::CMAKE) - { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"cmake", "0"}; - } - return get_path(paths, CMakeProvider(), abiToolVersionHandling == RequireExactVersions::YES); - } - if (tool == Tools::GIT) - { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"git", "0"}; - } - return get_path(paths, GitProvider()); - } - if (tool == Tools::NINJA) - { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"ninja", "0"}; - } - return get_path(paths, NinjaProvider()); - } - if (tool == Tools::POWERSHELL_CORE) + return xml_config_contents.get_lazy( + [this]() { return this->fs.read_contents(this->xml_config, VCPKG_LINE_INFO); }); + } + + virtual const Path& get_tool_path(StringView tool) const override { return get_tool_pathversion(tool).p; } + + static constexpr StringLiteral s_env_vcpkg_force_system_binaries = "VCPKG_FORCE_SYSTEM_BINARIES"; + + PathAndVersion get_path(const ToolProvider& tool) const + { + const bool env_force_system_binaries = + get_environment_variable(s_env_vcpkg_force_system_binaries).has_value(); + const bool env_force_download_binaries = + get_environment_variable("VCPKG_FORCE_DOWNLOADED_BINARIES").has_value(); + const auto maybe_tool_data = + parse_tool_data_from_xml(get_config_contents(), xml_config, tool.tool_data_name()); + + const bool download_available = maybe_tool_data.has_value() && !maybe_tool_data.get()->url.empty(); + // search for system searchable tools unless forcing downloads and download available + const bool consider_system = + tool.is_system_searchable() && !(env_force_download_binaries && download_available); + // search for downloaded tools unless forcing system search + const bool consider_downloads = !env_force_system_binaries || !consider_system; + + const bool exact_version = tool.is_abi_sensitive() && abiToolVersionHandling == RequireExactVersions::YES; + // forcing system search also disables version detection + const bool ignore_version = env_force_system_binaries; + + std::vector candidate_paths; + std::array min_version = tool.default_min_version(); + + if (auto tool_data = maybe_tool_data.get()) + { + // If there is an entry for the tool in vcpkgTools.xml, use that version as the minimum + min_version = tool_data->version; + + if (consider_downloads) { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"pwsh", "0"}; - } - return get_path( - paths, PowerShellCoreProvider(), abiToolVersionHandling == RequireExactVersions::YES); + // If we would consider downloading the tool, prefer the downloaded copy + candidate_paths.push_back(tool_data->exe_path(tools)); } - if (tool == Tools::NUGET) return get_path(paths, NuGetProvider()); - if (tool == Tools::ARIA2) return get_path(paths, Aria2Provider()); - if (tool == Tools::NODE) return get_path(paths, NodeProvider()); - if (tool == Tools::IFW_INSTALLER_BASE) return get_path(paths, IfwInstallerBaseProvider()); - if (tool == Tools::MONO) return get_path(paths, MonoProvider()); - if (tool == Tools::GSUTIL) + } + + if (consider_system) + { + // If we are considering system copies, first search the PATH, then search any special system locations + // (e.g Program Files). + auto paths_from_path = fs.find_from_PATH(tool.system_exe_stem()); + candidate_paths.insert(candidate_paths.end(), paths_from_path.cbegin(), paths_from_path.cend()); + + tool.add_system_paths(candidate_paths); + } + + if (ignore_version) + { + // If we are forcing the system copy (and therefore ignoring versions), take the first entry that + // exists. + const auto it = std::find_if(candidate_paths.begin(), candidate_paths.end(), [this](const Path& p) { + return this->fs.exists(p, IgnoreErrors{}); + }); + if (it != candidate_paths.end()) { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"gsutil", "0"}; - } - return get_path(paths, GsutilProvider()); + return {*it, "0"}; } - if (tool == Tools::AWSCLI) + } + else + { + // Otherwise, execute each entry and compare its version against the constraint. Take the first that + // matches. + const auto maybe_path = find_first_with_sufficient_version( + tool, candidate_paths, [&min_version, exact_version](const std::array& actual_version) { + if (exact_version) + { + return actual_version[0] == min_version[0] && actual_version[1] == min_version[1] && + actual_version[2] == min_version[2]; + } + return actual_version[0] > min_version[0] || + (actual_version[0] == min_version[0] && actual_version[1] > min_version[1]) || + (actual_version[0] == min_version[0] && actual_version[1] == min_version[1] && + actual_version[2] >= min_version[2]); + }); + if (const auto p = maybe_path.get()) { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"aws", "0"}; - } - return get_path(paths, AwsCliProvider()); + return *p; } - if (tool == Tools::COSCLI) + } + + if (consider_downloads) + { + // If none of the current entries are acceptable, fall back to downloading if possible + if (auto tool_data = maybe_tool_data.get()) { - if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - return {"cos", "0"}; - } - return get_path(paths, CosCliProvider()); + auto downloaded_path = download_tool(*tool_data); + auto downloaded_version = tool.get_version(*this, downloaded_path).value_or_exit(VCPKG_LINE_INFO); + return {std::move(downloaded_path), std::move(downloaded_version)}; } - if (tool == Tools::TAR) - { - auto tars = paths.get_filesystem().find_from_PATH("tar"); - if (tars.empty()) - { - Checks::exit_with_message(VCPKG_LINE_INFO, -#if defined(_WIN32) - "Could not find tar; the action you tried to take assumes Windows 10 " - "or later which are bundled with tar.exe." -#else // ^^^ _WIN32 // !_WIN32 vvv - "Could not find tar; please install it from your system package manager." -#endif // ^^^ !_WIN32 - - ); - } + } - return {tars[0], {}}; - } + // If no acceptable tool was found and downloading was unavailable, emit an error message + LocalizedString s = msg::format(msg::msgErrorMessage); + s.append(msgToolFetchFailed, msg::tool_name = tool.tool_data_name()); + if (env_force_system_binaries && download_available) + { + s.append_raw(" ").append(msgDownloadAvailable, msg::env_var = s_env_vcpkg_force_system_binaries); + } + if (consider_system) + { + tool.add_system_package_info(s); + } + else if (!download_available) + { + s.append_raw(" ").append(msgUnknownTool); + } + Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO, s); + } - // For other tools, we simply always auto-download them. - auto maybe_tool_data = parse_tool_data_from_xml(paths, tool); - if (auto p_tool_data = maybe_tool_data.get()) + const PathAndVersion& get_tool_pathversion(StringView tool) const + { + return path_version_cache.get_lazy(tool, [&]() -> PathAndVersion { + // First deal with specially handled tools. + // For these we may look in locations like Program Files, the PATH etc as well as the auto-downloaded + // location. + if (tool == Tools::CMAKE) return get_path(CMakeProvider()); + if (tool == Tools::GIT) return get_path(GitProvider()); + if (tool == Tools::NINJA) return get_path(NinjaProvider()); + if (tool == Tools::POWERSHELL_CORE) return get_path(PowerShellCoreProvider()); + if (tool == Tools::NUGET) return get_path(NuGetProvider()); + if (tool == Tools::ARIA2) return get_path(Aria2Provider()); + if (tool == Tools::NODE) return get_path(NodeProvider()); + if (tool == Tools::IFW_INSTALLER_BASE) return get_path(IfwInstallerBaseProvider()); + if (tool == Tools::MONO) return get_path(MonoProvider()); + if (tool == Tools::GSUTIL) return get_path(GsutilProvider()); + if (tool == Tools::AWSCLI) return get_path(AwsCliProvider()); + if (tool == Tools::COSCLI) return get_path(CosCliProvider()); + if (tool == Tools::TAR) { - if (paths.get_filesystem().exists(p_tool_data->exe_path, IgnoreErrors{})) - { - return {p_tool_data->exe_path, p_tool_data->sha512}; - } - return {fetch_tool(paths, tool, *p_tool_data), p_tool_data->sha512}; + return {find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), {}}; } - - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO, "Unknown or unavailable tool: %s", tool); + GenericToolProvider provider{tool}; + return get_path(provider); }); } - virtual const std::string& get_tool_version(const VcpkgPaths& paths, StringView tool) const override + virtual const std::string& get_tool_version(StringView tool) const override { - return get_tool_pathversion(paths, tool).version; + return get_tool_pathversion(tool).version; } }; - std::unique_ptr get_tool_cache(RequireExactVersions abiToolVersionHandling) + ExpectedL find_system_tar(const Filesystem& fs) + { + const auto tools = fs.find_from_PATH(Tools::TAR); + if (tools.empty()) + { + return msg::format(msg::msgErrorMessage) + .append(msgToolFetchFailed, msg::tool_name = Tools::TAR) +#if defined(_WIN32) + .append(msgToolInWin10) +#else + .append(msgInstallWithSystemManager) +#endif + ; + } + else + { + return tools[0]; + } + } + + std::unique_ptr get_tool_cache(Filesystem& fs, + std::shared_ptr downloader, + Path downloads, + Path xml_config, + Path tools, + RequireExactVersions abiToolVersionHandling) { - return std::make_unique(abiToolVersionHandling); + return std::make_unique( + fs, std::move(downloader), downloads, xml_config, tools, abiToolVersionHandling); } } diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index e8120aa2c6..50061d686c 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -379,14 +379,13 @@ namespace vcpkg , m_cache_root(default_registries_cache_path().value_or_exit(VCPKG_LINE_INFO)) , m_manifest_dir(compute_manifest_dir(fs, args, original_cwd)) , m_bundle(load_bundle_file(fs, root)) - , m_tool_cache(get_tool_cache(args.exact_abi_tools_versions.value_or(false) ? RequireExactVersions::YES - : RequireExactVersions::NO)) - , m_download_manager( - parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO)) + , m_download_manager(std::make_shared( + parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO))) , m_builtin_ports(process_output_directory(fs, args.builtin_ports_root_dir.get(), root / "ports")) , m_default_vs_path(args.default_visual_studio_path ? fs.almost_canonical(*args.default_visual_studio_path, VCPKG_LINE_INFO) : Path{}) + , scripts(process_input_directory(fs, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO)) { Debug::print("Bundle config: readonly=", m_bundle.m_readonly, @@ -404,10 +403,10 @@ namespace vcpkg const Path m_cache_root; const Path m_manifest_dir; const BundleSettings m_bundle; - const std::unique_ptr m_tool_cache; - const DownloadManager m_download_manager; + const std::shared_ptr m_download_manager; const Path m_builtin_ports; const Path m_default_vs_path; + const Path scripts; }; static Optional compute_installed(Filesystem& fs, @@ -432,6 +431,27 @@ namespace vcpkg return nullopt; } + static Path compute_downloads_root(const Filesystem& fs, + const VcpkgCmdArguments& args, + const Path& root, + const details::BundleSettings& bundle) + { + Path ret; + if (args.downloads_root_dir) + { + ret = *args.downloads_root_dir; + } + else if (bundle.m_readonly) + { + ret = get_platform_cache_home().value_or_exit(VCPKG_LINE_INFO) / "vcpkg" / "downloads"; + } + else + { + ret = root / "downloads"; + } + return fs.almost_canonical(ret, VCPKG_LINE_INFO); + } + struct VcpkgPathsImpl : VcpkgPathsImplStage1 { VcpkgPathsImpl(Filesystem& fs, const VcpkgCmdArguments& args, const Path& root, const Path& original_cwd) @@ -442,6 +462,8 @@ namespace vcpkg , m_registries_work_tree_dir(m_cache_root / "git") , m_registries_dot_git_dir(m_cache_root / "git" / ".git") , m_registries_git_trees(m_cache_root / "git-trees") + , downloads(compute_downloads_root(fs, args, root, m_bundle)) + , tools(downloads / "tools") , m_installed(compute_installed(fs, args, root, m_manifest_dir, m_bundle)) , buildtrees(maybe_get_tmp_path(fs, m_bundle, @@ -459,6 +481,13 @@ namespace vcpkg "packages", "pkgs", VCPKG_LINE_INFO)) + , m_tool_cache(get_tool_cache(fs, + m_download_manager, + downloads, + scripts / "vcpkgTools.xml", + tools, + args.exact_abi_tools_versions.value_or(false) ? RequireExactVersions::YES + : RequireExactVersions::NO)) , m_env_cache(m_ff_settings.compiler_tracking) , triplets_dirs(Util::fmap(args.overlay_triplets, [&fs](const std::string& p) { return fs.almost_canonical(p, VCPKG_LINE_INFO); @@ -508,9 +537,12 @@ namespace vcpkg const Path m_registries_work_tree_dir; const Path m_registries_dot_git_dir; const Path m_registries_git_trees; + const Path downloads; + const Path tools; const Optional m_installed; const Optional buildtrees; const Optional packages; + const std::unique_ptr m_tool_cache; Build::EnvCache m_env_cache; std::vector triplets_dirs; @@ -608,41 +640,20 @@ namespace vcpkg #endif } - static Path compute_downloads_root(const Filesystem& fs, - const VcpkgCmdArguments& args, - const Path& root, - const details::BundleSettings& bundle) - { - Path ret; - if (args.downloads_root_dir) - { - ret = *args.downloads_root_dir; - } - else if (bundle.m_readonly) - { - ret = get_platform_cache_home().value_or_exit(VCPKG_LINE_INFO) / "vcpkg" / "downloads"; - } - else - { - ret = root / "downloads"; - } - return fs.almost_canonical(ret, VCPKG_LINE_INFO); - } - VcpkgPaths::VcpkgPaths(Filesystem& filesystem, const VcpkgCmdArguments& args) : original_cwd(preferred_current_path(filesystem)) , root(determine_root(filesystem, original_cwd, args)) // this is used during the initialization of the below public members , m_pimpl(std::make_unique(filesystem, args, root, original_cwd)) + , scripts(m_pimpl->scripts) + , downloads(m_pimpl->downloads) + , tools(m_pimpl->tools) , builtin_registry_versions( process_output_directory(filesystem, args.builtin_registry_versions_dir.get(), root / "versions")) - , scripts(process_input_directory(filesystem, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO)) , prefab(root / "prefab") , buildsystems(scripts / "buildsystems") , buildsystems_msbuild_targets(buildsystems / "msbuild" / "vcpkg.targets") , buildsystems_msbuild_props(buildsystems / "msbuild" / "vcpkg.props") - , downloads(compute_downloads_root(filesystem, args, root, m_pimpl->m_bundle)) - , tools(downloads / "tools") , ports_cmake(filesystem.almost_canonical(scripts / "ports.cmake", VCPKG_LINE_INFO)) , triplets(filesystem.almost_canonical(root / "triplets", VCPKG_LINE_INFO)) , community_triplets(filesystem.almost_canonical(triplets / "community", VCPKG_LINE_INFO)) @@ -901,13 +912,11 @@ namespace vcpkg }); } - const Path& VcpkgPaths::get_tool_exe(StringView tool) const - { - return m_pimpl->m_tool_cache->get_tool_path(*this, tool); - } + const ToolCache& VcpkgPaths::get_tool_cache() const { return *m_pimpl->m_tool_cache; } + const Path& VcpkgPaths::get_tool_exe(StringView tool) const { return m_pimpl->m_tool_cache->get_tool_path(tool); } const std::string& VcpkgPaths::get_tool_version(StringView tool) const { - return m_pimpl->m_tool_cache->get_tool_version(*this, tool); + return m_pimpl->m_tool_cache->get_tool_version(tool); } GitConfig VcpkgPaths::git_builtin_config() const @@ -1303,7 +1312,7 @@ namespace vcpkg Checks::check_exit(VCPKG_LINE_INFO, m_pimpl->m_registry_set != nullptr); return *m_pimpl->m_registry_set; } - const DownloadManager& VcpkgPaths::get_download_manager() const { return m_pimpl->m_download_manager; } + const DownloadManager& VcpkgPaths::get_download_manager() const { return *m_pimpl->m_download_manager.get(); } DECLARE_AND_REGISTER_MESSAGE(ErrorVcvarsUnsupported, (msg::triplet),