Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: update regex tokenizer (en-du-it) #271

Merged
merged 1 commit into from
Feb 9, 2023
Merged

feat: update regex tokenizer (en-du-it) #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/tokenizer/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ export type Tokenizer = (
) => string[];

const splitRegex: Record<Language, RegExp> = {
dutch: /[^a-z0-9_'-]+/gim,
english: /[^a-z0-9_'-]+/gim,
dutch: /[^A-Za-zàèéìòóù0-9_'-]+/gim,
english: /[^A-Za-zàèéìòóù0-9_'-]+/gim,
french: /[^a-z0-9äâàéèëêïîöôùüûœç-]+/gim,
italian: /[^a-z0-9_'-]+/gim,
italian: /[^A-Za-zàèéìòóù0-9_'-]+/gim,
norwegian: /[^a-z0-9_æøåÆØÅäÄöÖüÜ]+/gim,
portuguese: /[^a-z0-9à-úÀ-Ú]/gim,
russian: /[^a-z0-9а-яА-ЯёЁ]+/gim,
Expand Down
4 changes: 2 additions & 2 deletions tests/lyra.dataset.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ t.test("lyra.dataset", async t => {
properties: ["description"],
});

t.equal(s1.count, 31285);
t.equal(s2.count, 28746);
t.equal(s1.count, 31294);
t.equal(s2.count, 28747);
t.equal(s3.count, 33644);
});

Expand Down
57 changes: 57 additions & 0 deletions tests/lyra.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -969,3 +969,60 @@ t.test("should search numbers in supported languages", async t => {

t.end();
});

// Tests for https://github.com/LyraSearch/lyra/issues/230
t.test("should correctly search accented words in Italian", async t => {
const db = await create({
schema: {
description: "string",
},
defaultLanguage: "italian",
});

await insert(db, {
description: "Il mio nome è Josè",
});

const searchResult = await search(db, {
term: "jose",
});
t.equal(searchResult.count, 1);
});

// Tests for https://github.com/LyraSearch/lyra/issues/230
t.test("should correctly search accented words in English", async t => {
const db = await create({
schema: {
description: "string",
},
defaultLanguage: "english",
});

await insert(db, {
description: "My name is Josè",
});

const searchResult = await search(db, {
term: "jose",
});
t.equal(searchResult.count, 1);
});

// Tests for https://github.com/LyraSearch/lyra/issues/230
t.test("should correctly search accented words in Dutch", async t => {
const db = await create({
schema: {
description: "string",
},
defaultLanguage: "dutch",
});

await insert(db, {
description: "Mein Name ist Josè",
});

const searchResult = await search(db, {
term: "jose",
});
t.equal(searchResult.count, 1);
});
60 changes: 30 additions & 30 deletions tests/snapshots/events.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"hits": [
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-89",
"description": "Social War:",
Expand All @@ -17,7 +17,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-57",
"description": "Gallic Wars:",
Expand All @@ -30,7 +30,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-55",
"description": "Gallic War",
Expand All @@ -43,7 +43,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-54",
"description": "Gallic Wars",
Expand All @@ -56,7 +56,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-53",
"description": "Parthian war:",
Expand All @@ -69,7 +69,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-53",
"description": "Gallic War:",
Expand All @@ -82,7 +82,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-48",
"description": "Civil War:",
Expand All @@ -95,7 +95,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-47",
"description": "Civil War:",
Expand All @@ -108,7 +108,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "-46",
"description": "Civil War:",
Expand All @@ -121,7 +121,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "1809/03/13",
"description": "Peninsular War",
Expand All @@ -139,7 +139,7 @@
"hits": [
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "1867/12/02",
"description": "Paraguayan War.",
Expand All @@ -152,7 +152,7 @@
},
{
"id": "",
"score": 4.922700520038653,
"score": 4.921722598465424,
"document": {
"date": "1950/02/14",
"description": " Cold War:",
Expand All @@ -165,7 +165,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "-86",
"description": "First Mithridatic War",
Expand All @@ -178,7 +178,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "632/01/27",
"description": "Ridda Wars begins",
Expand All @@ -191,7 +191,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "988",
"description": "Rus'–Byzantine War",
Expand All @@ -204,7 +204,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "1043/10/31",
"description": "Rus'-Byzantine War (1043).",
Expand All @@ -217,7 +217,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "1861/04/27",
"description": " American Civil War:",
Expand All @@ -230,7 +230,7 @@
},
{
"id": "",
"score": 4.351484115222462,
"score": 4.349997355979656,
"document": {
"date": "1914/12/24",
"description": " World War I:",
Expand All @@ -243,7 +243,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "630",
"description": "The Byzantine-Arab Wars begin.",
Expand All @@ -256,7 +256,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1522/12/20",
"description": "The Habsburg-Valois Wars begin.",
Expand All @@ -274,7 +274,7 @@
"hits": [
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1728/10/20",
"description": "The Meerkat–Mongoose war.",
Expand All @@ -287,7 +287,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1830/12/20",
"description": "The Java War ends.",
Expand All @@ -300,7 +300,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1862/09/17",
"description": " American Civil War ampndash",
Expand All @@ -313,7 +313,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1879/01/11",
"description": " The Anglo-Zulu War begins.",
Expand All @@ -326,7 +326,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1919/02/14",
"description": " The Polish-Soviet War begins.",
Expand All @@ -339,7 +339,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "1939/03/23",
"description": " The Slovak-Hungarian War begins.",
Expand All @@ -352,7 +352,7 @@
},
{
"id": "",
"score": 3.8624582297875905,
"score": 3.860623950530306,
"document": {
"date": "2003/02/05",
"description": "War in Darfur begins.",
Expand All @@ -365,7 +365,7 @@
},
{
"id": "",
"score": 3.4524120362550765,
"score": 3.4503660467084463,
"document": {
"date": "-156",
"description": "The first Dalmatian war begins.",
Expand All @@ -378,7 +378,7 @@
},
{
"id": "",
"score": 3.4524120362550765,
"score": 3.4503660467084463,
"document": {
"date": "-119",
"description": "The second Dalmatian war begins.",
Expand All @@ -391,7 +391,7 @@
},
{
"id": "",
"score": 3.4524120362550765,
"score": 3.4503660467084463,
"document": {
"date": "-78",
"description": "The Third Dalmatian war begins.",
Expand Down