Merge 49ed125 into b21a4b0

Author: stebloev

ydb/core/base/fulltext.cpp

@@ -1,4 +1,7 @@
 #include "fulltext.h"
+
+#include <contrib/libs/snowball/include/libstemmer.h>
+
 #include <util/charset/utf8.h>
 #include <util/generic/xrange.h>
 
@@ -172,7 +175,28 @@ namespace {
             return false;
         }
 
-        if (settings.has_language()) {
+        if (settings.use_filter_snowball()) {
+            if (settings.use_filter_ngram() || settings.use_filter_edge_ngram()) {
+                error = "cannot set use_filter_snowball with use_filter_ngam or use_filter_edge_ngram at the same time";
+                return false;
+            }
+
+            if (!settings.has_language()) {
+                error = "language required when use_filter_snowball is set";
+                return false;
+            }
+
+            bool supportedLanguage = false;
+            for (auto ptr = sb_stemmer_list(); *ptr != nullptr; ++ptr) {
+                if (settings.language() == *ptr) {
+                    supportedLanguage = true;
+                }
+            }
+            if (!supportedLanguage) {
+                error = "language is not supported by snowball";
+                return false;
+            }
+        } else if (settings.has_language()) {
             error = "Unsupported language setting";
             return false;
         }
@@ -268,6 +292,20 @@ TVector<TString> Analyze(const TString& text, const Ydb::Table::FulltextIndexSet
         }), tokens.end());
     }
 
+    if (settings.use_filter_snowball()) {
+        struct sb_stemmer* stemmer = sb_stemmer_new(settings.language().c_str(), nullptr);
+        for (auto& token : tokens) {
+            const sb_symbol* stemmed = sb_stemmer_stem(
+                stemmer,
+                reinterpret_cast<const sb_symbol*>(token.data()),
+                token.size()
+            );
+
+            const size_t resultLength = sb_stemmer_length(stemmer);
+            token = std::string(reinterpret_cast<const char*>(stemmed), resultLength);
+        }
+    }
+
     if (settings.use_filter_ngram() || settings.use_filter_edge_ngram()) {
         TVector<TString> ngrams;
         for (const auto& token : tokens) {
@@ -367,6 +405,8 @@ bool FillSetting(Ydb::Table::FulltextIndexSettings& settings, const TString& nam
         analyzers->set_filter_length_min(ParseInt32(name, value, error));
     } else if (nameLower == "filter_length_max") {
         analyzers->set_filter_length_max(ParseInt32(name, value, error));
+    } else if (nameLower == "use_filter_snowball") {
+        analyzers->set_use_filter_snowball(ParseBool(name, value, error));
     } else {
         error = TStringBuilder() << "Unknown index setting: " << name;
         return false;

ydb/core/base/ut/fulltext_ut.cpp

@@ -270,6 +270,22 @@ Y_UNIT_TEST_SUITE(NFulltext) {
         analyzers.set_filter_ngram_max_length(3);
         UNIT_ASSERT_VALUES_EQUAL(Analyze(text, analyzers), (TVector<TString>{"эт", "это", "те", "тек"}));
     }
+
+    Y_UNIT_TEST(AnalyzeFilterSnowball) {
+        Ydb::Table::FulltextIndexSettings::Analyzers analyzers;
+        analyzers.set_tokenizer(Ydb::Table::FulltextIndexSettings::WHITESPACE);
+        const TString russianText = "машины ездят по дорогам исправно";
+
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(russianText, analyzers), (TVector<TString>{"машины", "ездят", "по", "дорогам", "исправно"}));
+
+        analyzers.set_use_filter_snowball(true);
+        analyzers.set_language("russian");
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(russianText, analyzers), (TVector<TString>{"машин", "езд", "по", "дорог", "исправн"}));
+
+        const TString englishText = "cars are driving properly on the roads";
+        analyzers.set_language("english");
+        UNIT_ASSERT_VALUES_EQUAL(Analyze(englishText, analyzers), (TVector<TString>{"car", "are", "drive", "proper", "on", "the", "road"}));
+    }
 }
 
 }

ydb/core/base/ya.make

@@ -88,6 +88,7 @@ SRCS(
 )
 
 PEERDIR(
+    contrib/libs/snowball
     ydb/library/actors/core
     ydb/library/actors/helpers
     ydb/library/actors/interconnect

ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp

@@ -91,6 +91,23 @@ void AddIndexCovered(NQuery::TQueryClient& db) {
     UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
 }
 
+void AddIndexSnowball(NQuery::TQueryClient& db) {
+    TString query = R"sql(
+        ALTER TABLE `/Root/Texts` ADD INDEX fulltext_idx
+            GLOBAL USING fulltext
+            ON (Text)
+            WITH (
+                layout=flat,
+                tokenizer=standard,
+                use_filter_lowercase=true,
+                use_filter_snowball=true,
+                language=english
+            )
+    )sql";
+    auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync();
+    UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+}
+
 TResultSet ReadIndex(NQuery::TQueryClient& db) {
     TString query = R"sql(
         SELECT * FROM `/Root/Texts/fulltext_idx/indexImplTable`;
@@ -224,6 +241,31 @@ Y_UNIT_TEST(AddIndexEdgeNGram) {
     ])", NYdb::FormatResultSetYson(index));
 }
 
+Y_UNIT_TEST(AddIndexSnowball) {
+    auto kikimr = Kikimr();
+    auto db = kikimr.GetQueryClient();
+
+    CreateTexts(db);
+    UpsertTexts(db);
+    AddIndexSnowball(db);
+    const auto index = ReadIndex(db);
+    CompareYson(R"([
+        [[[100u];"anim"];
+        [[100u];"cat"];
+        [[200u];"cat"];
+        [[300u];"cat"];
+        [[100u];"chase"];
+        [[200u];"chase"];
+        [[200u];"dog"];
+        [[400u];"dog"];
+        [[400u];"fox"];
+        [[300u];"love"];
+        [[400u];"love"];
+        [[100u];"small"];
+        [[200u];"small"]
+    ])", NYdb::FormatResultSetYson(index));
+}
+
 Y_UNIT_TEST(InsertRow) {
     auto kikimr = Kikimr();
     auto db = kikimr.GetQueryClient();

ydb/public/api/protos/ydb_table.proto

@@ -229,6 +229,10 @@ message FulltextIndexSettings {
         // Maximum token length to keep (inclusive)
         // Must be used with use_filter_length
         optional int32 filter_length_max = 132 [(Ydb.value) = ">= 0"];
+
+        // Wether to apply stemming for each token
+        // TODO
+        optional bool use_filter_snowball = 140;
     }
 
     // Represents text analyzers settings for a specific column