Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Catalog search without support for Xapian boolean operators #1137

Merged
merged 3 commits into from
Sep 25, 2024
Merged

Catalog search without support for Xapian boolean operators #1137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90dd1cb
Enhanced test book descriptions with xapian keywords
veloman-yunkan Sep 25, 2024
de64a5a
Testing of Xapian query operators in catalog search
veloman-yunkan Sep 25, 2024
01bda6b
Disabled Xapian boolean operators in catalog query
veloman-yunkan Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/library.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,8 +645,6 @@ Xapian::Query buildXapianQueryFromFilterQuery(const Filter& filter)
//queryParser.set_stemmer(Xapian::Stem(iso639_3ToXapian(???)));
//queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_SOME);
const auto flags = Xapian::QueryParser::FLAG_PHRASE
| Xapian::QueryParser::FLAG_BOOLEAN
| Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE
| Xapian::QueryParser::FLAG_LOVEHATE
| Xapian::QueryParser::FLAG_WILDCARD
| partialQueryFlag;
Expand Down
6 changes: 3 additions & 3 deletions test/data/library.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
path="./zimfile_raycharles.zim"
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile_raycharles.zim"
title="Ray Charles"
description="Wikipedia articles about Ray Charles"
description="Wikipedia articles about Ray Charles (not all of them but near to what an average newborn may find more than enough)"
language="eng"
creator="Wikipedia"
publisher="Kiwix"
Expand All @@ -22,7 +22,7 @@
path="./zimfile_raycharles_uncategorized.zim"
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile_raycharles_uncategorized.zim"
title="Ray (uncategorized) Charles"
description="No category is assigned to this library entry."
description="No category is assigned to this library entry (neither adj nor xor was considered a good option)"
language="rus,eng"
creator="Wikipedia"
publisher="Kiwix"
Expand All @@ -39,7 +39,7 @@
path="./zimfile&other.zim"
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim"
title="Charles, Ray"
description="Wikipedia articles about Ray Charles"
description="Wikipedia articles about Ray Charles or why and when one should go to library"
language="fra"
creator="Wikipedia"
publisher="Kiwix"
Expand Down
185 changes: 173 additions & 12 deletions test/library_server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ std::string maskVariableOPDSFeedData(std::string s)
#define _CHARLES_RAY_CATALOG_ENTRY(CONTENT_NAME) CATALOG_ENTRY( \
"charlesray", \
"Charles, Ray", \
"Wikipedia articles about Ray Charles", \
"Wikipedia articles about Ray Charles or why and when one should go to library", \
"fra", \
"wikipedia_fr_ray_charles",\
"jazz",\
Expand All @@ -120,7 +120,7 @@ std::string maskVariableOPDSFeedData(std::string s)
#define _RAY_CHARLES_CATALOG_ENTRY(CONTENT_NAME) CATALOG_ENTRY(\
"raycharles",\
"Ray Charles",\
"Wikipedia articles about Ray Charles",\
"Wikipedia articles about Ray Charles (not all of them but near to what an average newborn may find more than enough)",\
"eng",\
"wikipedia_en_ray_charles",\
"wikipedia",\
Expand All @@ -139,7 +139,7 @@ std::string maskVariableOPDSFeedData(std::string s)
#define UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY CATALOG_ENTRY(\
"raycharles_uncategorized",\
"Ray (uncategorized) Charles",\
"No category is assigned to this library entry.",\
"No category is assigned to this library entry (neither adj nor xor was considered a good option)",\
"rus,eng",\
"wikipedia_ru_ray_charles",\
"",\
Expand Down Expand Up @@ -199,8 +199,8 @@ TEST_F(LibraryServerTest, catalog_search_by_phrase)
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
CATALOG_LINK_TAGS
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
Expand All @@ -218,8 +218,8 @@ TEST_F(LibraryServerTest, catalog_search_by_words)
" <startIndex>0</startIndex>\n"
" <itemsPerPage>3</itemsPerPage>\n"
CATALOG_LINK_TAGS
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
Expand All @@ -239,8 +239,8 @@ TEST_F(LibraryServerTest, catalog_prefix_search)
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
CATALOG_LINK_TAGS
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
Expand Down Expand Up @@ -275,8 +275,8 @@ TEST_F(LibraryServerTest, catalog_search_with_word_exclusion)
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
CATALOG_LINK_TAGS
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
Expand Down Expand Up @@ -331,8 +331,8 @@ TEST_F(LibraryServerTest, catalog_search_by_category)
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
CATALOG_LINK_TAGS
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
Expand Down Expand Up @@ -772,10 +772,171 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms)
" <totalResults>2</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}

TEST_F(LibraryServerTest, catalog_v2_entries_filtering_special_queries)
{
{
// 'or' doesn't act as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=Or");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=Or")
" <title>Filtered Entries (q=Or)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
CHARLES_RAY_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'and' doesn't act as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=and");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=and")
" <title>Filtered Entries (q=and)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
CHARLES_RAY_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'not' doesn't act as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=not");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=not")
" <title>Filtered Entries (q=not)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'xor' doesn't act as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=xor");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=xor")
" <title>Filtered Entries (q=xor)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'or' acts as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=wikipedia%20or%20library");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=wikipedia%20or%20library")
" <title>Filtered Entries (q=wikipedia%20or%20library)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
CHARLES_RAY_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'and' acts as a Xapian boolean operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=wikipedia%20and%20articles");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=wikipedia%20and%20articles")
" <title>Filtered Entries (q=wikipedia%20and%20articles)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
CHARLES_RAY_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'near' doesn't act as a Xapian query operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=near");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=near")
" <title>Filtered Entries (q=near)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'adj' doesn't act as a Xapian query operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=adj");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=adj")
" <title>Filtered Entries (q=adj)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>1</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>1</itemsPerPage>\n"
UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}

{
// 'near' doesn't act as a Xapian query operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=charles%20near%20why");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=charles%20near%20why")
" <title>Filtered Entries (q=charles%20near%20why)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>0</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>0</itemsPerPage>\n"
"</feed>\n"
);
}

{
// 'adj' doesn't act as a Xapian query operator
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries?q=charles%20adj%20why");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("?q=charles%20adj%20why")
" <title>Filtered Entries (q=charles%20adj%20why)</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <totalResults>0</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>0</itemsPerPage>\n"
"</feed>\n"
);
}
}

TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_language)
Expand Down Expand Up @@ -841,8 +1002,8 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_category)
" <totalResults>2</totalResults>\n"
" <startIndex>0</startIndex>\n"
" <itemsPerPage>2</itemsPerPage>\n"
RAY_CHARLES_CATALOG_ENTRY
CHARLES_RAY_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
Expand Down Expand Up @@ -1086,7 +1247,7 @@ TEST_F(LibraryServerTest, no_name_mapper_catalog_v2_individual_entry_access)
" <div class=\"book__header\">\n" \
" <div id=\"book__title\">Charles, Ray</div>\n" \
" </div>\n" \
" <div class=\"book__description\" title=\"Wikipedia articles about Ray Charles\">Wikipedia articles about Ray Charles</div>\n" \
" <div class=\"book__description\" title=\"Wikipedia articles about Ray Charles or why and when one should go to library\">Wikipedia articles about Ray Charles or why and when one should go to library</div>\n" \
" </div>\n" \
" </a>\n" \
" <div class=\"book__meta\">\n" \
Expand All @@ -1113,7 +1274,7 @@ TEST_F(LibraryServerTest, no_name_mapper_catalog_v2_individual_entry_access)
" <div class=\"book__header\">\n" \
" <div id=\"book__title\">Ray Charles</div>\n" \
" </div>\n" \
" <div class=\"book__description\" title=\"Wikipedia articles about Ray Charles\">Wikipedia articles about Ray Charles</div>\n" \
" <div class=\"book__description\" title=\"Wikipedia articles about Ray Charles (not all of them but near to what an average newborn may find more than enough)\">Wikipedia articles about Ray Charles (not all of them but near to what an average newborn may find more than enough)</div>\n" \
" </div>\n" \
" </a>\n" \
" <div class=\"book__meta\">\n" \
Expand All @@ -1140,7 +1301,7 @@ TEST_F(LibraryServerTest, no_name_mapper_catalog_v2_individual_entry_access)
" <div class=\"book__header\">\n" \
" <div id=\"book__title\">Ray (uncategorized) Charles</div>\n" \
" </div>\n" \
" <div class=\"book__description\" title=\"No category is assigned to this library entry.\">No category is assigned to this library entry.</div>\n" \
" <div class=\"book__description\" title=\"No category is assigned to this library entry (neither adj nor xor was considered a good option)\">No category is assigned to this library entry (neither adj nor xor was considered a good option)</div>\n" \
" </div>\n" \
" </a>\n" \
" <div class=\"book__meta\">\n" \
Expand Down
Loading