Create DB 1.1 schema for system reference strings (#564)

The primary motivation of this change was to add new fields for use in the list and upgrade commands; system reference strings that can be used to associate a winget package with software currently installed on a machine.  This required creation of schema 1.1, as well as some refactoring to make inheritance and behavior changes easier between versions.

The two types of strings added are PackageFamilyName for MSIX packages, and ProductCode for packages that register through ARP (Add/Remove Programs).  These are added with the option of having multiple of each in the event that a package's installers have unique values.  The values are stored with their casing folded to allow for ordinal comparisons, a performance requirement due to future work on list and upgrade.

Additional noteworthy changes:
1. Changed from implicit indices to explicit ones in 1.1, and drop more of them during preparation for packaging. This should result in a ~35% reduction in page count in the database, with no impact to performance.
2. Added a dev debug tool in SQLiteWrapper.cpp; set WINGET_SQLITE_EXPLAIN_QUERY_PLAN_ENABLED to 1 to have the query plan for every SQLite prepared statement output to logging.

src/AppInstallerCLITests/Strings.cpp

@@ -4,6 +4,7 @@
 #include "TestCommon.h"
 #include <AppInstallerStrings.h>
 
+using namespace std::string_view_literals;
 using namespace AppInstaller::Utility;
 
 
@@ -124,3 +125,11 @@ TEST_CASE("CaseInsensitiveStartsWith", "[strings]")
     REQUIRE(!CaseInsensitiveStartsWith("withstarts", "starts"));
     REQUIRE(!CaseInsensitiveStartsWith(" starts", "starts"));
 }
+
+TEST_CASE("FoldCase", "[strings]")
+{
+    REQUIRE(FoldCase(""sv) == FoldCase(""sv));
+    REQUIRE(FoldCase("foldcase"sv) == FoldCase("FOLDCASE"sv));
+    REQUIRE(FoldCase(u8"f\xF6ldcase"sv) == FoldCase(u8"F\xD6LDCASE"sv));
+    REQUIRE(FoldCase(u8"foldc\x430se"sv) == FoldCase(u8"FOLDC\x410SE"sv));
+}

src/AppInstallerCommonCore/AppInstallerStrings.cpp

@@ -295,6 +295,55 @@ namespace AppInstaller::Utility
         return result;
     }
 
+    std::string FoldCase(std::string_view input)
+    {
+        if (input.empty())
+        {
+            return {};
+        }
+
+        wil::unique_any<UCaseMap*, decltype(ucasemap_close), &ucasemap_close> caseMap;
+        UErrorCode errorCode = UErrorCode::U_ZERO_ERROR;
+        caseMap.reset(ucasemap_open(nullptr, U_FOLD_CASE_DEFAULT, &errorCode));
+
+        if (U_FAILURE(errorCode))
+        {
+            AICLI_LOG(Core, Error, << "ucasemap_open returned " << errorCode);
+            THROW_HR(E_UNEXPECTED);
+        }
+
+        int32_t cch = ucasemap_utf8FoldCase(caseMap.get(), nullptr, 0, input.data(), static_cast<int32_t>(input.size()), &errorCode);
+        if (errorCode != U_BUFFER_OVERFLOW_ERROR)
+        {
+            AICLI_LOG(Core, Error, << "ucasemap_utf8FoldCase returned " << errorCode);
+            THROW_HR(E_UNEXPECTED);
+        }
+
+        errorCode = UErrorCode::U_ZERO_ERROR;
+
+        std::string result(cch, '\0');
+        cch = ucasemap_utf8FoldCase(caseMap.get(), &result[0], cch, input.data(), static_cast<int32_t>(input.size()), &errorCode);
+        if (U_FAILURE(errorCode))
+        {
+            AICLI_LOG(Core, Error, << "ucasemap_utf8FoldCase returned " << errorCode);
+            THROW_HR(E_UNEXPECTED);
+        }
+
+        while (result.back() == '\0')
+        {
+            result.pop_back();
+        }
+
+        return result;
+    }
+
+    NormalizedString FoldCase(const NormalizedString& input)
+    {
+        NormalizedString result;
+        result.assign(FoldCase(static_cast<std::string_view>(input)));
+        return result;
+    }
+
     bool IsEmptyOrWhitespace(std::wstring_view str)
     {
         if (str.empty())

src/AppInstallerCommonCore/Public/AppInstallerStrings.h

@@ -96,6 +96,14 @@ namespace AppInstaller::Utility
     // Get the lower case version of the given std::wstring
     std::wstring ToLower(std::wstring_view in);
 
+    // Folds the case of the given std::string
+    // See https://unicode-org.github.io/icu/userguide/transforms/casemappings.html#case-folding
+    std::string FoldCase(std::string_view input);
+
+    // Folds the case of the given NormalizedString, returning it as also Normalized
+    // See https://unicode-org.github.io/icu/userguide/transforms/casemappings.html#case-folding
+    NormalizedString FoldCase(const NormalizedString& input);
+
     // Checks if the input string is empty or whitespace
     bool IsEmptyOrWhitespace(std::wstring_view str);
 
@@ -106,9 +114,6 @@ namespace AppInstaller::Utility
     // Removes whitespace from the beginning and end of the string.
     std::string& Trim(std::string& str);
 
-    // Gets the number of bytes remaining in the stream.
-    std::streamsize GetRemainingByteCount(std::istream& stream);
-
     // Reads the entire stream into a string.
     std::string ReadEntireStream(std::istream& stream);
 }

src/AppInstallerCommonCore/Public/winget/ManifestInstaller.h

@@ -66,6 +66,12 @@ namespace AppInstaller::Manifest
         // Store Product Id
         string_t ProductId;
 
+        // Package family name for MSIX packaged installers.
+        string_t PackageFamilyName;
+
+        // Product code for ARP (Add/Remove Programs) installers.
+        string_t ProductCode;
+
         // If present, has more precedence than root
         InstallerTypeEnum InstallerType;
 

src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj

@@ -188,6 +188,10 @@
     <ClInclude Include="Microsoft\Schema\1_0\SearchResultsTable.h" />
     <ClInclude Include="Microsoft\Schema\1_0\TagsTable.h" />
     <ClInclude Include="Microsoft\Schema\1_0\VersionTable.h" />
+    <ClInclude Include="Microsoft\Schema\1_1\Interface.h" />
+    <ClInclude Include="Microsoft\Schema\1_1\PackageFamilyNameTable.h" />
+    <ClInclude Include="Microsoft\Schema\1_1\ProductCodeTable.h" />
+    <ClInclude Include="Microsoft\Schema\1_1\SearchResultsTable.h" />
     <ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" />
     <ClInclude Include="Microsoft\Schema\MetadataTable.h" />
     <ClInclude Include="Microsoft\Schema\Version.h" />
@@ -214,12 +218,14 @@
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader>
     </ClCompile>
     <ClCompile Include="Microsoft\PreIndexedPackageSourceFactory.cpp" />
-    <ClCompile Include="Microsoft\Schema\1_0\Interface.cpp" />
+    <ClCompile Include="Microsoft\Schema\1_0\Interface_1_0.cpp" />
     <ClCompile Include="Microsoft\Schema\1_0\ManifestTable.cpp" />
     <ClCompile Include="Microsoft\Schema\1_0\OneToManyTable.cpp" />
     <ClCompile Include="Microsoft\Schema\1_0\OneToOneTable.cpp" />
     <ClCompile Include="Microsoft\Schema\1_0\PathPartTable.cpp" />
-    <ClCompile Include="Microsoft\Schema\1_0\SearchResultsTable.cpp" />
+    <ClCompile Include="Microsoft\Schema\1_0\SearchResultsTable_1_0.cpp" />
+    <ClCompile Include="Microsoft\Schema\1_1\Interface_1_1.cpp" />
+    <ClCompile Include="Microsoft\Schema\1_1\SearchResultsTable_1_1.cpp" />
     <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" />
     <ClCompile Include="Microsoft\Schema\Version.cpp" />
     <ClCompile Include="Microsoft\SQLiteIndex.cpp" />

src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters

@@ -28,6 +28,9 @@
     <Filter Include="ICU">
       <UniqueIdentifier>{dac1a359-45ac-4456-8a83-f7df10058197}</UniqueIdentifier>
     </Filter>
+    <Filter Include="Microsoft\Schema\1_1">
+      <UniqueIdentifier>{aef8989c-44fd-4848-ae2c-c81d3f2e2c72}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="pch.h">
@@ -114,6 +117,18 @@
     <ClInclude Include="AggregatedSource.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="Microsoft\Schema\1_1\Interface.h">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClInclude>
+    <ClInclude Include="Microsoft\Schema\1_1\PackageFamilyNameTable.h">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClInclude>
+    <ClInclude Include="Microsoft\Schema\1_1\ProductCodeTable.h">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClInclude>
+    <ClInclude Include="Microsoft\Schema\1_1\SearchResultsTable.h">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="pch.cpp">
@@ -131,9 +146,6 @@
     <ClCompile Include="Microsoft\Schema\Version.cpp">
       <Filter>Microsoft\Schema</Filter>
     </ClCompile>
-    <ClCompile Include="Microsoft\Schema\1_0\Interface.cpp">
-      <Filter>Microsoft\Schema\1_0</Filter>
-    </ClCompile>
     <ClCompile Include="Microsoft\Schema\1_0\OneToOneTable.cpp">
       <Filter>Microsoft\Schema\1_0</Filter>
     </ClCompile>
@@ -161,15 +173,24 @@
     <ClCompile Include="SQLiteTempTable.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="Microsoft\Schema\1_0\SearchResultsTable.cpp">
-      <Filter>Microsoft\Schema\1_0</Filter>
-    </ClCompile>
     <ClCompile Include="ICU\SQLiteICU.c">
       <Filter>ICU</Filter>
     </ClCompile>
     <ClCompile Include="AggregatedSource.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="Microsoft\Schema\1_0\Interface_1_0.cpp">
+      <Filter>Microsoft\Schema\1_0</Filter>
+    </ClCompile>
+    <ClCompile Include="Microsoft\Schema\1_1\Interface_1_1.cpp">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClCompile>
+    <ClCompile Include="Microsoft\Schema\1_0\SearchResultsTable_1_0.cpp">
+      <Filter>Microsoft\Schema\1_0</Filter>
+    </ClCompile>
+    <ClCompile Include="Microsoft\Schema\1_1\SearchResultsTable_1_1.cpp">
+      <Filter>Microsoft\Schema\1_1</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp

@@ -124,6 +124,13 @@ namespace AppInstaller::Repository::Microsoft
         m_version = m_interface->GetVersion();
     }
 
+#ifndef AICLI_DISABLE_TEST_HOOKS
+    void SQLiteIndex::ForceVersion(const Schema::Version& version)
+    {
+        m_interface = version.CreateISQLiteIndex();
+    }
+#endif
+
     void SQLiteIndex::AddManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath)
     {
         AICLI_LOG(Repo, Info, << "Adding manifest from file [" << manifestPath << "]");
@@ -159,7 +166,7 @@ namespace AppInstaller::Repository::Microsoft
 
         SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_updatemanifest");
 
-        bool result = m_interface->UpdateManifest(m_dbconn, manifest, relativePath);
+        bool result = m_interface->UpdateManifest(m_dbconn, manifest, relativePath).first;
 
         if (result)
         {

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h

@@ -51,6 +51,12 @@ namespace AppInstaller::Repository::Microsoft
         // Gets the schema version of the index.
         Schema::Version GetVersion() const { return m_version; }
 
+#ifndef AICLI_DISABLE_TEST_HOOKS
+        // Changes the version of the interface being used to operate on the database.
+        // Should only be used for testing.
+        void ForceVersion(const Schema::Version& version);
+#endif
+
         // Gets the last write time for the index.
         std::chrono::system_clock::time_point GetLastWriteTime();
 

src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h

@@ -2,24 +2,38 @@
 // Licensed under the MIT License.
 #pragma once
 #include "Microsoft/Schema/ISQLiteIndex.h"
+#include "Microsoft/Schema/1_0/SearchResultsTable.h"
+
+#include <memory>
+#include <vector>
 
 
 namespace AppInstaller::Repository::Microsoft::Schema::V1_0
 {
     // Interface to this schema version exposed through ISQLiteIndex.
-    struct Interface : public SQLiteIndexBase
+    struct Interface : public ISQLiteIndex
     {
         // Version 1.0
         Schema::Version GetVersion() const override;
         void CreateTables(SQLite::Connection& connection) override;
-        void AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
-        bool UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
-        void RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
+        SQLite::rowid_t AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
+        std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
+        SQLite::rowid_t RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override;
         void PrepareForPackaging(SQLite::Connection& connection) override;
         SearchResult Search(SQLite::Connection& connection, const SearchRequest& request) override;
         std::optional<std::string> GetIdStringById(SQLite::Connection& connection, SQLite::rowid_t id) override;
         std::optional<std::string> GetNameStringById(SQLite::Connection& connection, SQLite::rowid_t id) override;
         std::optional<std::string> GetPathStringByKey(SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) override;
         std::vector<Utility::VersionAndChannel> GetVersionsById(SQLite::Connection& connection, SQLite::rowid_t id) override;
+
+    protected:
+        // Creates the search results table.
+        virtual std::unique_ptr<SearchResultsTable> CreateSearchResultsTable(SQLite::Connection& connection) const;
+
+        // Gets the ordering of matches to execute, with more specific matches coming first.
+        virtual std::vector<MatchType> GetMatchTypeOrder(MatchType type) const;
+
+        // Executes all relevant searchs for the query.
+        virtual void PerformQuerySearch(SearchResultsTable& resultsTable, const RequestMatch& query) const;
     };
 }