Skip to content

Commit 14b9947

Browse files
ViatorusViatorus
authored
feat(full-text-search): add full-text-search as separate package (#35)
1 parent 67d61ac commit 14b9947

28 files changed

+5176
-7
lines changed

‎.travis.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,16 @@ addons:
1919
- google-chrome
2020
packages:
2121
- google-chrome-stable
22+
- oracle-java8-set-default
2223
- python3
2324
- python3-pip
2425

2526
before_install:
27+
- curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb && sudo service elasticsearch start
2628
- pip3 install mkdocs
2729
- export CHROME_BIN=chromium-browser
2830
- export DISPLAY=:99.0
2931
- sh -e /etc/init.d/xvfb start
30-
3132
script:
3233
- npm run lint || travis_terminate 1
3334
- npm test || travis_terminate 1

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ The following is the list of supported scopes:
140140
* **local-storage**: The local storage adapter.
141141
* **indexed-storage**: The indexed db storage adapter.
142142
* **fs-storage**: The file system storage adapter.
143-
* **fts**: A full text search for the database.
143+
* **full-text-search**: The full text search engine.
144144

145145
There are currently a few exceptions to the "use package name" rule:
146146

‎config/build.js

+13-5
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ const stream = require("stream");
88
const conventionalChangelog = require("conventional-changelog");
99

1010
const PACKAGES = [
11-
"loki",
12-
"partitioning-adapter",
13-
"local-storage",
14-
"indexed-storage",
15-
"fs-storage"
11+
// "loki",
12+
// "partitioning-adapter",
13+
// "local-storage",
14+
// "indexed-storage",
15+
// "fs-storage",
16+
"full-text-search"
1617
];
1718

1819
const ROOT_DIR = process.cwd();
@@ -194,6 +195,13 @@ function build() {
194195
}
195196
}
196197
}
198+
if (json.optionalDependencies) {
199+
for (let pack of Object.keys(json.optionalDependencies)) {
200+
if (pack.startsWith("@lokijs/")) {
201+
json.optionalDependencies[pack] = VERSION;
202+
}
203+
}
204+
}
197205
fs.writeFileSync(`/${NPM_DIR}/package.json`, JSON.stringify(json, null, 2));
198206
}
199207
}

‎config/eslintrc.js

+1
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,6 @@ module.exports = {
2323
"always"
2424
],
2525
"arrow-parens": [2, "always"],
26+
"no-constant-condition": 0
2627
}
2728
};

‎package-lock.json

+46
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
"babel-register": "^6.26.0",
3636
"conventional-changelog": "^1.1.5",
3737
"coveralls": "^2.13.1",
38+
"elasticsearch": "^13.3.1",
3839
"eslint": "^4.6.1",
3940
"eslint-loader": "^1.9.0",
4041
"istanbul-instrumenter-loader": "^3.0.0",

‎packages/full-text-search/package.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "@lokijs/full-text-search",
3+
"description": "A full text search engine.",
4+
"author": "Various authors",
5+
"license": "MIT",
6+
"repository": {
7+
"type": "git",
8+
"url": "https://github.com/LokiJS-Forge/LokiJS2.git"
9+
},
10+
"main": "lokijs.full-text-search.js",
11+
"optionalDependencies": {
12+
"@lokijs/loki": "0"
13+
}
14+
}

‎packages/full-text-search/spec/generic/inverted_index.spec.js

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/* global describe, it, expect */
2+
import {InvertedIndex} from "../../src/inverted_index";
3+
4+
describe("inverted index", () => {
5+
6+
let ii = new InvertedIndex();
7+
8+
let field1 = "Hello world, how are you today?!";
9+
let field2 = "Well done world...";
10+
let field3 = "I am good, and you?";
11+
let field4 = "Now again inside today! You...";
12+
let field5 = "Good bye NO! for all worlds...";
13+
14+
it ("get", () => {
15+
expect(ii.documentCount).toBeNumber();
16+
expect(ii.documentStore).toBeObject();
17+
expect(ii.totalFieldLength).toBeNumber();
18+
expect(ii.tokenizer).toBeObject();
19+
expect(ii.root).toBeObject();
20+
});
21+
22+
it("insert", () => {
23+
ii.insert(field1, 1);
24+
expect(() =>ii.insert(field2, 1)).toThrowErrorOfType("Error");
25+
ii.insert(field3, 2);
26+
27+
ii.tokenizer.add("bad_tokenizer", () => [""]);
28+
ii.insert(field4, 3);
29+
ii.tokenizer.remove("bad_tokenizer");
30+
ii.insert(field4, 4);
31+
ii.insert(field5, 5);
32+
});
33+
34+
it("remove", () => {
35+
ii.remove(1);
36+
ii.remove(4);
37+
ii.remove(15);
38+
});
39+
40+
it("getTermIndex", () => {
41+
expect(InvertedIndex.getTermIndex("you", ii.root)).not.toBe(null);
42+
expect(InvertedIndex.getTermIndex("ayou", ii.root, 1)).not.toBe(null);
43+
expect(InvertedIndex.getTermIndex("you", ii.root, 10)).toBe(null);
44+
expect(InvertedIndex.getTermIndex("xyz1234", ii.root)).toBe(null);
45+
});
46+
47+
it("getNextTermIndex", () => {
48+
InvertedIndex.getNextTermIndex(ii.root);
49+
let idx = InvertedIndex.getTermIndex("you", ii.root);
50+
expect(InvertedIndex.getNextTermIndex(idx)).not.toBe(null);
51+
});
52+
53+
it("extendTermIndex", () => {
54+
expect(InvertedIndex.extendTermIndex(ii.root)).toBeArray();
55+
});
56+
57+
it("serialize", () => {
58+
let ii1 = new InvertedIndex();
59+
ii1.insert(field1, 1);
60+
ii1.insert(field2, 2);
61+
ii1.insert(field3, 3);
62+
63+
let ii2 = new InvertedIndex();
64+
ii2.insert(field1, 1);
65+
ii2.insert(field2, 2);
66+
ii2.insert(field3, 3);
67+
ii2.insert(field4, 4);
68+
69+
let ii3 = InvertedIndex.fromJSONObject(JSON.parse(JSON.stringify(ii2)));
70+
71+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
72+
ii2.remove(4);
73+
ii3.remove(4);
74+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
75+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
76+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
77+
78+
ii2.remove(1);
79+
ii3.remove(2);
80+
expect(JSON.stringify(ii2)).not.toEqual(JSON.stringify(ii1));
81+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii1));
82+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii2));
83+
84+
ii1.remove(1);
85+
ii1.remove(2);
86+
ii2.remove(2);
87+
ii3.remove(1);
88+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
89+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
90+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
91+
92+
ii2 = InvertedIndex.fromJSONObject(JSON.parse(JSON.stringify(ii1)));
93+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
94+
95+
ii1.insert(field5, 5);
96+
expect(JSON.stringify(ii2)).not.toEqual(JSON.stringify(ii1));
97+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii1));
98+
99+
ii1.remove(5);
100+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
101+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
102+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
103+
104+
// Check if still can be used
105+
ii3.insert(field5, 6);
106+
ii3.remove(6);
107+
});
108+
109+
it("serialize without optimization", () => {
110+
let ii1 = new InvertedIndex({optimizeChanges: false});
111+
ii1.insert(field1, 1);
112+
ii1.insert(field2, 2);
113+
ii1.insert(field3, 3);
114+
115+
let ii2 = new InvertedIndex({optimizeChanges: false});
116+
ii2.insert(field1, 1);
117+
ii2.insert(field2, 2);
118+
ii2.insert(field3, 3);
119+
ii2.insert(field4, 4);
120+
121+
let ii3 = InvertedIndex.fromJSONObject(JSON.parse(JSON.stringify(ii2)));
122+
123+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
124+
ii2.remove(4);
125+
ii3.remove(4);
126+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
127+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
128+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
129+
130+
ii2.remove(1);
131+
ii3.remove(2);
132+
expect(JSON.stringify(ii2)).not.toEqual(JSON.stringify(ii1));
133+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii1));
134+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii2));
135+
136+
// Compare with optimized inverted index.
137+
let iio3 = new InvertedIndex();
138+
iio3.insert(field1, 1);
139+
iio3.insert(field3, 3);
140+
expect(JSON.stringify(ii3.root)).toEqual(JSON.stringify(iio3.root));
141+
142+
ii1.remove(1);
143+
ii1.remove(2);
144+
ii2.remove(2);
145+
ii3.remove(1);
146+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
147+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
148+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
149+
150+
ii2 = InvertedIndex.fromJSONObject(JSON.parse(JSON.stringify(ii1)));
151+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
152+
153+
ii1.insert(field5, 5);
154+
expect(JSON.stringify(ii2)).not.toEqual(JSON.stringify(ii1));
155+
expect(JSON.stringify(ii3)).not.toEqual(JSON.stringify(ii1));
156+
157+
ii1.remove(5);
158+
expect(JSON.stringify(ii2)).toEqual(JSON.stringify(ii1));
159+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii1));
160+
expect(JSON.stringify(ii3)).toEqual(JSON.stringify(ii2));
161+
162+
// Check if still can be used
163+
ii3.insert(field5, 6);
164+
ii3.remove(6);
165+
});
166+
});

‎packages/full-text-search/spec/generic/languages/de.js

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import {DE} from "../../../src/language/de";
2+
3+
export const de = {
4+
tokenizer: DE,
5+
docs: [
6+
"An Deutschland grenzen neun Nachbarländer und naturräumlich im Norden die Gewässer der Nord- und Ostsee, im Süden das Bergland der Alpen. Es liegt in der gemäßigten Klimazone, zählt mit rund 80 Millionen Einwohnern zu den dicht besiedelten Flächenstaaten und gilt international als das Land mit der dritthöchsten Zahl von Einwanderern. aufeinanderfolgenden. auffassen.",
7+
"Deutschland als Urlaubsziel verfügt über günstige Voraussetzungen: Gebirgslandschaften (Alpen und Mittelgebirge), See- und Flusslandschaften, die Küsten und Inseln der Nord- und Ostsee, zahlreiche Kulturdenkmäler und eine Vielzahl geschichtsträchtiger Städte sowie gut ausgebaute Infrastruktur. Vorteilhaft ist die zentrale Lage in Europa."
8+
],
9+
tests: [{
10+
what: "find the word",
11+
search: "deutschland",
12+
found: [0, 1]
13+
}, {
14+
what: "find the word",
15+
search: "urlaubsziel",
16+
found: [1]
17+
}, {
18+
what: "find the word",
19+
search: "gewass",
20+
found: [0]
21+
}, {
22+
what: "find the word",
23+
search: "verfugt",
24+
found: [1]
25+
}, {
26+
what: "never find a word that does not exist, like",
27+
search: "inexistent",
28+
found: []
29+
}, {
30+
what: "never find a stop word like",
31+
search: "und",
32+
found: []
33+
}, {
34+
what: "find a correctly stemmed word",
35+
search: "auffassung",
36+
found: [0]
37+
}]
38+
};

0 commit comments

Comments
 (0)