chore(lyra): export tokenizer function

oramasearch · Aug 26, 2022 · 3e6df9a · 3e6df9a
1 parent 99eb188
commit 3e6df9a
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 3 deletions.
diff --git a/src/lyra.ts b/src/lyra.ts
@@ -10,6 +10,7 @@ import { trackInsertion } from "./insertion-checker";
 type Index = Record<string, Node>;
 
 export { formatNanoseconds } from "./utils";
+export { tokenize } from "./tokenizer";
 
 export type PropertyType = "string" | "number" | "boolean";
 

diff --git a/src/tokenizer/index.ts b/src/tokenizer/index.ts
@@ -13,15 +13,21 @@ const splitRegex: Record<Language, RegExp> = {
   swedish: /[^a-z0-9_åÅäÄöÖüÜ-]+/gim,
 };
 
-export function tokenize(input: string, language: Language = "english") {
+export function tokenize(input: string, language: Language = "english", allowDuplicates = false) {
   /* c8 ignore next 3 */
   if (typeof input !== "string") {
     return [input];
   }
 
   const splitRule = splitRegex[language];
   const tokens = input.toLowerCase().split(splitRule).map(replaceDiacritics);
-  return Array.from(new Set(trim(tokens)));
+  const trimTokens = trim(tokens);
+
+  if (!allowDuplicates) {
+    return Array.from(new Set(trimTokens));
+  }
+
+  return trimTokens;
 }
 
 function trim(text: string[]): string[] {

diff --git a/tap-snapshots/tests/tokenizer.test.ts.test.cjs b/tap-snapshots/tests/tokenizer.test.ts.test.cjs
@@ -45,6 +45,24 @@ Array [
 ]
 `
 
+exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in english and allow duplicates > Should tokenize and stem correctly in english and allow duplicates-O1 1`] = `
+Array [
+  "this",
+  "is",
+  "a",
+  "test",
+  "with",
+  "duplicates",
+]
+`
+
+exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in english and allow duplicates > Should tokenize and stem correctly in english and allow duplicates-O2 1`] = `
+Array [
+  "it's",
+  "alive",
+]
+`
+
 exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in french > Should tokenize and stem correctly in french-O1 1`] = `
 Array [
   "voyons",

diff --git a/tests/tokenizer.test.ts b/tests/tokenizer.test.ts
@@ -2,7 +2,7 @@ import t from "tap";
 import { tokenize } from "../src/tokenizer";
 
 t.test("Tokenizer", t => {
-  t.plan(9);
+  t.plan(10);
 
   t.test("Should tokenize and stem correctly in english", t => {
     t.plan(2);
@@ -17,6 +17,19 @@ t.test("Tokenizer", t => {
     t.matchSnapshot(O2, `${t.name}-O2`);
   });
 
+  t.test("Should tokenize and stem correctly in english and allow duplicates", t => {
+    t.plan(2);
+
+    const I1 = "this is a test with test duplicates";
+    const I2 = "it's alive! it's alive!";
+
+    const O1 = tokenize(I1, "english");
+    const O2 = tokenize(I2, "english");
+
+    t.matchSnapshot(O1, `${t.name}-O1`);
+    t.matchSnapshot(O2, `${t.name}-O2`);
+  });
+
   t.test("Should tokenize and stem correctly in french", t => {
     t.plan(2);