Teach x-ci-verify-versions to check that versions exist in the database

include/vcpkg/base/expected.h

@@ -220,6 +220,12 @@ namespace vcpkg
             return std::move(*m_t.get());
         }
 
+        Error& error() &
+        {
+            unreachable_if_not_error(VCPKG_LINE_INFO);
+            return m_error;
+        }
+
         const Error& error() const&
         {
             unreachable_if_not_error(VCPKG_LINE_INFO);
@@ -232,6 +238,12 @@ namespace vcpkg
             return std::move(m_error);
         }
 
+        const Error&& error() const&&
+        {
+            unreachable_if_not_error(VCPKG_LINE_INFO);
+            return std::move(m_error);
+        }
+
         typename ExpectedHolder<T>::const_pointer get() const noexcept
         {
             if (value_is_error)

include/vcpkg/base/jsonreader.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/fwd/json.h>
+#include <vcpkg/base/fwd/parse.h>
 
 #include <vcpkg/base/chrono.h>
 #include <vcpkg/base/json.h>
@@ -42,29 +43,43 @@ namespace vcpkg::Json
         IDeserializer& operator=(IDeserializer&&) = default;
     };
 
+    struct ReaderMessage
+    {
+        MessageKind kind;
+        LocalizedString message;
+
+        ReaderMessage(MessageKind kind, LocalizedString&& message);
+        ReaderMessage(const ReaderMessage&);
+        ReaderMessage(ReaderMessage&&);
+        ReaderMessage& operator=(const ReaderMessage&);
+        ReaderMessage& operator=(ReaderMessage&&);
+    };
+
+    LocalizedString join(const std::vector<ReaderMessage>& messages);
+    LocalizedString flatten_reader_messages(const std::vector<ReaderMessage>& messages, MessageSink& warningsSink);
+
     struct Reader
     {
-        Reader();
+        explicit Reader(StringView origin);
 
-        const std::vector<LocalizedString>& errors() const { return m_errors; }
+        const std::vector<ReaderMessage>& messages() const { return m_messages; }
+        std::size_t error_count() const { return m_error_count; }
 
         void add_missing_field_error(const LocalizedString& type, StringView key, const LocalizedString& key_type);
         void add_expected_type_error(const LocalizedString& expected_type);
         void add_extra_field_error(const LocalizedString& type, StringView fields, StringView suggestion = {});
-        void add_generic_error(const LocalizedString& type, LocalizedString&& message);
+        void add_generic_error(const LocalizedString& type, StringView message);
 
         void add_warning(LocalizedString type, StringView msg);
 
-        const std::vector<LocalizedString>& warnings() const { return m_warnings; }
-
         std::string path() const noexcept;
 
     private:
         template<class Type>
         friend struct IDeserializer;
 
-        std::vector<LocalizedString> m_errors;
-        std::vector<LocalizedString> m_warnings;
+        std::vector<ReaderMessage> m_messages;
+        std::size_t m_error_count;
         struct JsonPathElement
         {
             constexpr JsonPathElement() = default;
@@ -88,6 +103,7 @@ namespace vcpkg::Json
             std::vector<JsonPathElement>& m_path;
         };
 
+        StringView m_origin;
         std::vector<JsonPathElement> m_path;
 
     public:

include/vcpkg/base/messages.h

@@ -84,27 +84,42 @@ namespace vcpkg
         static LocalizedString from_raw(std::basic_string<T>&& s) noexcept;
         static LocalizedString from_raw(StringView s);
 
-        LocalizedString& append_raw(char c);
-        LocalizedString& append_raw(StringView s);
+        LocalizedString& append_raw(char c) &;
+        LocalizedString&& append_raw(char c) &&;
+        LocalizedString& append_raw(StringView s) &;
+        LocalizedString&& append_raw(StringView s) &&;
         template<class T, class = decltype(std::declval<const T&>().to_string(std::declval<std::string&>()))>
-        LocalizedString& append_raw(const T& s)
+        LocalizedString& append_raw(const T& s) &
         {
             s.to_string(m_data);
             return *this;
         }
-        LocalizedString& append(const LocalizedString& s);
+        template<class T, class = decltype(std::declval<const T&>().to_string(std::declval<std::string&>()))>
+        LocalizedString&& append_raw(const T& s) &&
+        {
+            return std::move(append_raw(s));
+        }
+        LocalizedString& append(const LocalizedString& s) &;
+        LocalizedString&& append(const LocalizedString& s) &&;
         template<VCPKG_DECL_MSG_TEMPLATE>
-        LocalizedString& append(VCPKG_DECL_MSG_ARGS)
+        LocalizedString& append(VCPKG_DECL_MSG_ARGS) &
         {
             msg::format_to(*this, VCPKG_EXPAND_MSG_ARGS);
             return *this;
         }
-        LocalizedString& append_indent(size_t indent = 1);
+        template<VCPKG_DECL_MSG_TEMPLATE>
+        LocalizedString&& append(VCPKG_DECL_MSG_ARGS) &&
+        {
+            return std::move(append(VCPKG_EXPAND_MSG_ARGS));
+        }
+        LocalizedString& append_indent(size_t indent = 1) &;
+        LocalizedString&& append_indent(size_t indent = 1) &&;
 
         // 0 items - Does nothing
         // 1 item - .append_raw(' ').append(item)
         // 2+ items - foreach: .append_raw('\n').append_indent(indent).append(item)
-        LocalizedString& append_floating_list(int indent, View<LocalizedString> items);
+        LocalizedString& append_floating_list(int indent, View<LocalizedString> items) &;
+        LocalizedString&& append_floating_list(int indent, View<LocalizedString> items) &&;
         friend bool operator==(const LocalizedString& lhs, const LocalizedString& rhs) noexcept;
         friend bool operator!=(const LocalizedString& lhs, const LocalizedString& rhs) noexcept;
         friend bool operator<(const LocalizedString& lhs, const LocalizedString& rhs) noexcept;

include/vcpkg/base/strings.h

@@ -110,7 +110,7 @@ namespace vcpkg::Strings
         if (first != last)
         {
             Strings::append(output, transformer(*first));
-            for (++first; first != last; ++first)
+            while (++first != last)
             {
                 output.append(delimiter.data(), delimiter.size());
                 Strings::append(output, transformer(*first));

include/vcpkg/base/util.h

@@ -414,6 +414,12 @@ namespace vcpkg::Util
         return std::any_of(rng.begin(), rng.end(), std::move(pred));
     }
 
+    template<class Range, class Pred>
+    bool none_of(Range&& rng, Pred pred)
+    {
+        return std::none_of(rng.begin(), rng.end(), std::move(pred));
+    }
+
     template<class Range, class Comp = std::less<>>
     Range&& sort_unique_erase(Range&& cont, Comp comp = Comp())
     {

include/vcpkg/paragraphparser.h

@@ -49,11 +49,11 @@ namespace vcpkg
     using ParseExpected = vcpkg::ExpectedT<std::unique_ptr<P>, std::unique_ptr<ParseControlErrorInfo>>;
 
     template<class P>
-    ExpectedL<P> map_parse_expected_to_localized_string(ParseExpected<P>&& parse_expected)
+    ExpectedL<std::unique_ptr<P>> map_parse_expected_to_localized_string(ParseExpected<P>&& parse_expected)
     {
         if (auto value = parse_expected.get())
         {
-            return std::move(**value);
+            return std::move(*value);
         }
 
         return LocalizedString::from_raw(parse_expected.error()->to_string());

include/vcpkg/paragraphs.h

@@ -5,9 +5,13 @@
 #include <vcpkg/fwd/registries.h>
 
 #include <vcpkg/base/expected.h>
+#include <vcpkg/base/stringview.h>
 
 #include <vcpkg/sourceparagraph.h>
 
+#include <utility>
+#include <vector>
+
 namespace vcpkg::Paragraphs
 {
     uint64_t get_load_ports_stats();
@@ -22,11 +26,27 @@ namespace vcpkg::Paragraphs
 
     bool is_port_directory(const ReadOnlyFilesystem& fs, const Path& maybe_directory);
 
-    ParseExpected<SourceControlFile> try_load_port(const ReadOnlyFilesystem& fs, const Path& port_directory);
-    ParseExpected<SourceControlFile> try_load_port_text(const std::string& text,
-                                                        StringView origin,
-                                                        bool is_manifest,
-                                                        MessageSink& warning_sink);
+    struct PortLoadResult
+    {
+        ExpectedL<SourceControlFileAndLocation> maybe_scfl;
+        std::string on_disk_contents;
+    };
+
+    // If an error occurs, the Expected will be in the error state.
+    // Otherwise, if the port is known, the maybe_scfl->source_control_file contains the loaded port information.
+    // Otherwise, maybe_scfl->source_control_file is nullptr.
+    PortLoadResult try_load_port(const ReadOnlyFilesystem& fs, StringView port_name, const Path& port_directory);
+    // Identical to try_load_port, but the port unknown condition is mapped to an error.
+    PortLoadResult try_load_port_required(const ReadOnlyFilesystem& fs,
+                                          StringView port_name,
+                                          const Path& port_directory);
+    ExpectedL<SourceControlFileAndLocation> try_load_project_manifest_text(StringView text,
+                                                                           StringView origin,
+                                                                           MessageSink& warning_sink);
+    ExpectedL<SourceControlFileAndLocation> try_load_port_manifest_text(StringView text,
+                                                                        StringView origin,
+                                                                        MessageSink& warning_sink);
+    ExpectedL<SourceControlFileAndLocation> try_load_control_file_text(StringView text, StringView origin);
 
     ExpectedL<BinaryControlFile> try_load_cached_package(const ReadOnlyFilesystem& fs,
                                                          const Path& package_dir,
@@ -35,7 +55,7 @@ namespace vcpkg::Paragraphs
     struct LoadResults
     {
         std::vector<SourceControlFileAndLocation> paragraphs;
-        std::vector<std::unique_ptr<ParseControlErrorInfo>> errors;
+        std::vector<std::pair<std::string, LocalizedString>> errors;
     };
 
     LoadResults try_load_all_registry_ports(const ReadOnlyFilesystem& fs, const RegistrySet& registries);

include/vcpkg/registries.h

@@ -18,7 +18,6 @@
 #include <map>
 #include <memory>
 #include <string>
-#include <system_error>
 #include <vector>
 
 namespace vcpkg
@@ -76,19 +75,24 @@ namespace vcpkg
     {
         virtual StringLiteral kind() const = 0;
 
-        // returns nullptr if the port doesn't exist
+        // If an error occurs, the ExpectedL will be in an error state.
+        // Otherwise, if the port is known, returns a pointer to RegistryEntry describing the port.
+        // Otherwise, returns a nullptr unique_ptr.
         virtual ExpectedL<std::unique_ptr<RegistryEntry>> get_port_entry(StringView port_name) const = 0;
 
-        // appends the names of the ports to the out parameter
-        // may result in duplicated port names; make sure to Util::sort_unique_erase at the end
+        // Appends the names of the known ports to the out parameter.
+        // May result in duplicated port names; make sure to Util::sort_unique_erase at the end
         virtual ExpectedL<Unit> append_all_port_names(std::vector<std::string>& port_names) const = 0;
 
-        // appends the names of the ports to the out parameter if this can be known without
+        // Appends the names of the ports to the out parameter if this can be known without
         // network access.
-        // returns true if names were appended, otherwise returns false.
+        // Returns true iff names were appended.
         virtual ExpectedL<bool> try_append_all_port_names_no_network(std::vector<std::string>& port_names) const = 0;
 
-        virtual ExpectedL<Version> get_baseline_version(StringView port_name) const = 0;
+        // If an error occurs, the ExpectedL will be in an error state.
+        // Otherwise, if the port is in the baseline, returns the version that baseline denotes.
+        // Otherwise, the Optional is disengaged.
+        virtual ExpectedL<Optional<Version>> get_baseline_version(StringView port_name) const = 0;
 
         virtual ~RegistryImplementation() = default;
     };
@@ -130,7 +134,7 @@ namespace vcpkg
         // the returned list is sorted by priority.
         std::vector<const RegistryImplementation*> registries_for_port(StringView name) const;
 
-        ExpectedL<Version> baseline_for_port(StringView port_name) const;
+        ExpectedL<Optional<Version>> baseline_for_port(StringView port_name) const;
 
         View<Registry> registries() const { return registries_; }
 
@@ -170,24 +174,58 @@ namespace vcpkg
         std::string git_tree;
     };
 
-    ExpectedL<Optional<std::vector<GitVersionDbEntry>>> get_builtin_versions(const VcpkgPaths& paths,
-                                                                             StringView port_name);
+    struct GitVersionsLoadResult
+    {
+        // If the versions database file does not exist, a disengaged Optional
+        // Otherwise, if a file I/O error occurred or the file is malformed, that error
+        // Otherwise, the loaded version database records
+        ExpectedL<Optional<std::vector<GitVersionDbEntry>>> entries;
+        Path versions_file_path;
+    };
 
-    ExpectedL<std::map<std::string, Version, std::less<>>> get_builtin_baseline(const VcpkgPaths& paths);
+    GitVersionsLoadResult load_git_versions_file(const ReadOnlyFilesystem& fs,
+                                                 const Path& registry_versions,
+                                                 StringView port_name);
 
-    bool is_git_commit_sha(StringView sv);
+    struct FullGitVersionsDatabase
+    {
+        explicit FullGitVersionsDatabase(const ReadOnlyFilesystem& fs,
+                                         const Path& registry_versions,
+                                         std::map<std::string, GitVersionsLoadResult, std::less<>>&& initial);
+        FullGitVersionsDatabase(FullGitVersionsDatabase&&);
+        FullGitVersionsDatabase& operator=(FullGitVersionsDatabase&&);
 
-    // Returns the effective match length of the package pattern `pattern` against `name`.
-    // No match is 0, exact match is SIZE_MAX, wildcard match is the length of the pattern.
-    // Note that the * is included in the match size to distinguish from 0 == no match.
-    size_t package_pattern_match(StringView name, StringView pattern);
+        const GitVersionsLoadResult& lookup(StringView port_name);
+        const std::map<std::string, GitVersionsLoadResult, std::less<>>& cache() const;
+
+    private:
+        const ReadOnlyFilesystem* m_fs;
+        Path m_registry_versions;
+        std::map<std::string, GitVersionsLoadResult, std::less<>> m_cache;
+    };
+
+    // The outer expected only contains directory enumeration errors; individual parse errors are within
+    ExpectedL<FullGitVersionsDatabase> load_all_git_versions_files(const ReadOnlyFilesystem& fs,
+                                                                   const Path& registry_versions);
 
     struct FilesystemVersionDbEntry
     {
         SchemedVersion version;
         Path p;
     };
 
+    ExpectedL<Optional<std::vector<FilesystemVersionDbEntry>>> load_filesystem_versions_file(
+        const ReadOnlyFilesystem& fs, const Path& registry_versions, StringView port_name, const Path& registry_root);
+
+    ExpectedL<std::map<std::string, Version, std::less<>>> get_builtin_baseline(const VcpkgPaths& paths);
+
+    bool is_git_commit_sha(StringView sv);
+
+    // Returns the effective match length of the package pattern `pattern` against `name`.
+    // No match is 0, exact match is SIZE_MAX, wildcard match is the length of the pattern.
+    // Note that the * is included in the match size to distinguish from 0 == no match.
+    size_t package_pattern_match(StringView name, StringView pattern);
+
     std::unique_ptr<Json::IDeserializer<std::vector<GitVersionDbEntry>>> make_git_version_db_deserializer();
     std::unique_ptr<Json::IDeserializer<std::vector<FilesystemVersionDbEntry>>> make_filesystem_version_db_deserializer(
         const Path& root);