feat(full-text-search): english and german language support

* move language support in own packages
* add german and english language support
* optimize language class layout
* add unit tests

Author: Viatorus

config/build.js

@@ -13,7 +13,10 @@ const PACKAGES = [
   "local-storage",
   "indexed-storage",
   "fs-storage",
-  "full-text-search"
+  "full-text-search",
+  "full-text-search-language",
+  "full-text-search-language-de",
+  "full-text-search-language-en",
 ];
 
 const ROOT_DIR = process.cwd();

packages/full-text-search-language-de/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@lokijs/full-text-search-language-de",
+  "description": "A german language analyzer for the full-text-search.",
+  "author": "Various authors",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/LokiJS-Forge/LokiJS2.git"
+  },
+  "main": "lokijs.full-text-search-language-de.js",
+  "dependencies": {
+    "@lokijs/full-text-search": "0",
+    "@lokijs/full-text-search-language": "0"
+  }
+}

packages/full-text-search/spec/generic/languages/de.js packages/full-text-search-language-de/spec/generic/de.spec.ts

@@ -1,6 +1,7 @@
-import {DE} from "../../../src/language/de";
+import {DE} from "../../src/de";
+import {createLanguageTest, LanguageTestData} from "../../../full-text-search-language/spec/helper/create_lanuage_test";
 
-export const de = {
+export const de: LanguageTestData = {
   tokenizer: DE,
   docs: [
     "An Deutschland grenzen neun Nachbarländer und naturräumlich im Norden die Gewässer der Nord- und Ostsee, im Süden das Bergland der Alpen. Es liegt in der gemäßigten Klimazone, zählt mit rund 80 Millionen Einwohnern zu den dicht besiedelten Flächenstaaten und gilt international als das Land mit der dritthöchsten Zahl von Einwanderern. aufeinanderfolgenden. auffassen.",
@@ -9,30 +10,32 @@ export const de = {
   tests: [{
     what: "find the word",
     search: "deutschland",
-    found: [0, 1]
+    expected: [0, 1]
   }, {
     what: "find the word",
     search: "urlaubsziel",
-    found: [1]
+    expected: [1]
   }, {
     what: "find the word",
     search: "gewass",
-    found: [0]
+    expected: [0]
   }, {
     what: "find the word",
     search: "verfugt",
-    found: [1]
+    expected: [1]
   }, {
     what: "never find a word that does not exist, like",
     search: "inexistent",
-    found: []
+    expected: []
   }, {
     what: "never find a stop word like",
     search: "und",
-    found: []
+    expected: []
   }, {
     what: "find a correctly stemmed word",
-    search: "auffassung",
-    found: [0]
+    search: "auffass",
+    expected: [0]
   }]
 };
+
+createLanguageTest("de", de);

packages/full-text-search/src/language/de.js packages/full-text-search-language-de/src/de.ts

@@ -1,20 +1,23 @@
 /*
  * From MihaiValentin/lunr-languages.
- * Last update from 04/16/2017 - 19af41fb9bd644d9081ad274f96f700b21464290
+ * Last update from 2017/04/16 - 19af41fb9bd644d9081ad274f96f700b21464290
  */
-import {generateTrimmer, generateStopWordFilter, Among, SnowballProgram} from "./support";
-import {Tokenizer} from "../tokenizer";
+import {Tokenizer} from "../../full-text-search/src/index";
+import {
+  generateTrimmer,
+  generateStopWordFilter,
+  Among,
+  SnowballProgram
+} from "../../full-text-search-language/src/language";
 
-let wordCharacters = "A-Za-z\xAA\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02B8\u02E0-\u02E4\u1D00-\u1D25\u1D2C-\u1D5C\u1D62-\u1D65\u1D6B-\u1D77\u1D79-\u1DBE\u1E00-\u1EFF\u2071\u207F\u2090-\u209C\u212A\u212B\u2132\u214E\u2160-\u2188\u2C60-\u2C7F\uA722-\uA787\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA7FF\uAB30-\uAB5A\uAB5C-\uAB64\uFB00-\uFB06\uFF21-\uFF3A\uFF41-\uFF5A";
-let trimmer = generateTrimmer(wordCharacters);
+class GermanStemmer {
+  public getCurrent: () => string;
+  public setCurrent: (word: string) => void;
+  public stem: () => void;
 
-let tkz = new Tokenizer();
-
-tkz.add("trimmer-de", trimmer);
-
-let stemmer = ((() => {
-  /* create the wrapped stemmer object */
-  let st = new (function GermanStemmer() {
+  constructor() {
+    // Write everything in the constructor to reduce code size and increase performance.
+    // The original implementation uses a ES5 anonymous function class.
     let a_0 = [new Among("", -1, 6), new Among("U", 0, 2),
       new Among("Y", 0, 1), new Among("\u00E4", 0, 3),
       new Among("\u00F6", 0, 4), new Among("\u00FC", 0, 5)
@@ -53,16 +56,17 @@ let stemmer = ((() => {
       117, 30, 4
     ];
 
-    let I_x;
-    let I_p2;
-    let I_p1;
+    let I_x: number;
+    let I_p2: number;
+    let I_p1: number;
     let sbp = new SnowballProgram();
-    this.setCurrent = (word) => {
+
+    this.setCurrent = (word: string) => {
       sbp.setCurrent(word);
     };
     this.getCurrent = () => sbp.getCurrent();
 
-    function habr1(c1, c2, v_1) {
+    function habr1(c1: string, c2: string, v_1: number) {
       if (sbp.eq_s(1, c1)) {
         sbp.ket = sbp.cursor;
         if (sbp.in_grouping(g_v, 97, 252)) {
@@ -304,19 +308,12 @@ let stemmer = ((() => {
       r_standard_suffix();
       sbp.cursor = sbp.limit_backward;
       r_postlude();
-      return true;
     };
-  });
-
-  /* and return a function that stems a word for the current locale */
-  return (token) => {
-    st.setCurrent(token);
-    st.stem();
-    return st.getCurrent();
-  };
-}))();
+  }
+}
 
-tkz.setSplitter("whitespace-splitter", function defaultSplitter(str) {
+// Split at whitespace and dashes.
+function splitter(str: string) {
   let trimmedTokens = [];
   let tokens = str.split(/[\s-]+/);
   for (let i = 0; i < tokens.length; i++) {
@@ -325,11 +322,22 @@ tkz.setSplitter("whitespace-splitter", function defaultSplitter(str) {
     }
   }
   return trimmedTokens;
-});
+}
+
+const st = new GermanStemmer();
 
-tkz.add("stemmer-de", stemmer);
+function stemmer(token: string) {
+  st.setCurrent(token);
+  st.stem();
+  return st.getCurrent();
+}
 
-let stopWordFilter = generateStopWordFilter(["aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am", "an", "ander", "andere", "anderem", "anderen", "anderer", "anderes", "anderm", "andern", "anderr", "anders", "auch", "auf", "aus", "bei", "bin", "bis", "bist", "da", "damit", "dann", "das", "dasselbe", "dazu", "daß", "dein", "deine", "deinem", "deinen", "deiner", "deines", "dem", "demselben", "den", "denn", "denselben", "der", "derer", "derselbe", "derselben", "des", "desselben", "dessen", "dich", "die", "dies", "diese", "dieselbe", "dieselben", "diesem", "diesen", "dieser", "dieses", "dir", "doch", "dort", "du", "durch", "ein", "eine", "einem", "einen", "einer", "eines", "einig", "einige", "einigem", "einigen", "einiger", "einiges", "einmal", "er", "es", "etwas", "euch", "euer", "eure", "eurem", "euren", "eurer", "eures", "für", "gegen", "gewesen", "hab", "habe", "haben", "hat", "hatte", "hatten", "hier", "hin", "hinter", "ich", "ihm", "ihn", "ihnen", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "im", "in", "indem", "ins", "ist", "jede", "jedem", "jeden", "jeder", "jedes", "jene", "jenem", "jenen", "jener", "jenes", "jetzt", "kann", "kein", "keine", "keinem", "keinen", "keiner", "keines", "können", "könnte", "machen", "man", "manche", "manchem", "manchen", "mancher", "manches", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mich", "mir", "mit", "muss", "musste", "nach", "nicht", "nichts", "noch", "nun", "nur", "ob", "oder", "ohne", "sehr", "sein", "seine", "seinem", "seinen", "seiner", "seines", "selbst", "sich", "sie", "sind", "so", "solche", "solchem", "solchen", "solcher", "solches", "soll", "sollte", "sondern", "sonst", "um", "und", "uns", "unse", "unsem", "unsen", "unser", "unses", "unter", "viel", "vom", "von", "vor", "war", "waren", "warst", "was", "weg", "weil", "weiter", "welche", "welchem", "welchen", "welcher", "welches", "wenn", "werde", "werden", "wie", "wieder", "will", "wir", "wird", "wirst", "wo", "wollen", "wollte", "während", "würde", "würden", "zu", "zum", "zur", "zwar", "zwischen", "über"]);
-tkz.add("stopWordFilter-de", stopWordFilter);
+const trimmer = generateTrimmer("A-Za-z\xAA\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02B8\u02E0-\u02E4\u1D00-\u1D25\u1D2C-\u1D5C\u1D62-\u1D65\u1D6B-\u1D77\u1D79-\u1DBE\u1E00-\u1EFF\u2071\u207F\u2090-\u209C\u212A\u212B\u2132\u214E\u2160-\u2188\u2C60-\u2C7F\uA722-\uA787\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA7FF\uAB30-\uAB5A\uAB5C-\uAB64\uFB00-\uFB06\uFF21-\uFF3A\uFF41-\uFF5A");
+const stopWordFilter = generateStopWordFilter(["aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am", "an", "ander", "andere", "anderem", "anderen", "anderer", "anderes", "anderm", "andern", "anderr", "anders", "auch", "auf", "aus", "bei", "bin", "bis", "bist", "da", "damit", "dann", "das", "dasselbe", "dazu", "daß", "dein", "deine", "deinem", "deinen", "deiner", "deines", "dem", "demselben", "den", "denn", "denselben", "der", "derer", "derselbe", "derselben", "des", "desselben", "dessen", "dich", "die", "dies", "diese", "dieselbe", "dieselben", "diesem", "diesen", "dieser", "dieses", "dir", "doch", "dort", "du", "durch", "ein", "eine", "einem", "einen", "einer", "eines", "einig", "einige", "einigem", "einigen", "einiger", "einiges", "einmal", "er", "es", "etwas", "euch", "euer", "eure", "eurem", "euren", "eurer", "eures", "für", "gegen", "gewesen", "hab", "habe", "haben", "hat", "hatte", "hatten", "hier", "hin", "hinter", "ich", "ihm", "ihn", "ihnen", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "im", "in", "indem", "ins", "ist", "jede", "jedem", "jeden", "jeder", "jedes", "jene", "jenem", "jenen", "jener", "jenes", "jetzt", "kann", "kein", "keine", "keinem", "keinen", "keiner", "keines", "können", "könnte", "machen", "man", "manche", "manchem", "manchen", "mancher", "manches", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mich", "mir", "mit", "muss", "musste", "nach", "nicht", "nichts", "noch", "nun", "nur", "ob", "oder", "ohne", "sehr", "sein", "seine", "seinem", "seinen", "seiner", "seines", "selbst", "sich", "sie", "sind", "so", "solche", "solchem", "solchen", "solcher", "solches", "soll", "sollte", "sondern", "sonst", "um", "und", "uns", "unse", "unsem", "unsen", "unser", "unses", "unter", "viel", "vom", "von", "vor", "war", "waren", "warst", "was", "weg", "weil", "weiter", "welche", "welchem", "welchen", "welcher", "welches", "wenn", "werde", "werden", "wie", "wieder", "will", "wir", "wird", "wirst", "wo", "wollen", "wollte", "während", "würde", "würden", "zu", "zum", "zur", "zwar", "zwischen", "über"]);
 
-export {tkz as DE};
+// Create, configure and export the tokenizer.
+export const DE: Tokenizer = new Tokenizer();
+DE.setSplitter("whitespace-splitter", splitter);
+DE.add("trimmer-de", trimmer);
+DE.add("stemmer-de", stemmer);
+DE.add("stopWordFilter-de", stopWordFilter);

packages/full-text-search-language-de/webpack.config.js

@@ -0,0 +1,13 @@
+/* global __dirname, module, require */
+const path = require("path");
+const webpackConigCreator = require('../../config/webpack-config-creator.js');
+
+module.exports = webpackConigCreator({
+  entry: path.join(__dirname, "src", "de.ts"),
+  filename: "lokijs.full-text-search-language-de.js",
+  library: "@lokijs/full-text-search-language-de",
+  externals: {
+    "../../full-text-search-language/src/language": "@lokijs/full-text-search-language",
+    "../../full-text-search/src/index": "@lokijs/full-text-search"
+  },
+});

packages/full-text-search-language-en/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@lokijs/full-text-search-language-en",
+  "description": "An English language analyzer for the full-text-search.",
+  "author": "Various authors",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/LokiJS-Forge/LokiJS2.git"
+  },
+  "main": "lokijs.full-text-search-language-en.js",
+  "dependencies": {
+    "@lokijs/full-text-search": "0",
+    "@lokijs/full-text-search-language": "0"
+  }
+}

packages/full-text-search-language-en/spec/generic/en.spec.ts

@@ -0,0 +1,41 @@
+import {EN} from "../../src/en";
+import {createLanguageTest, LanguageTestData} from "../../../full-text-search-language/spec/helper/create_lanuage_test";
+
+export const en: LanguageTestData = {
+  tokenizer: EN,
+  docs: [
+    "In on announcing if of comparison pianoforte projection. Maids hoped gay yet bed asked blind dried point. On abroad danger likely regret twenty edward do. Too horrible consider followed may differed age.",
+    "By so delight of showing neither believe he present. Deal sigh up in shew away when. Pursuit considering express no or prepare replied."
+  ],
+  tests: [{
+    what: "find the word",
+    search: "announcing",
+    expected: [0]
+  },{
+    what: "find the word",
+    search: "believe",
+    expected: [1]
+  }, {
+    what: "find the word",
+    search: "consider",
+    expected: [0, 1]
+  }, {
+    what: "find the word",
+    search: "show",
+    expected: [1]
+  }, {
+    what: "never find a word that does not exist, like",
+    search: "inexistent",
+    expected: []
+  }, {
+    what: "never find a stop word like",
+    search: "neither",
+    expected: []
+  }, {
+    what: "find a correctly stemmed word",
+    search: "show",
+    expected: [1]
+  }]
+};
+
+createLanguageTest("en", en);