Speed up indexing, use stopwords, strip xml tags

.gitignore

@@ -79,3 +79,7 @@ jspm_packages
 
 # local development examples
 examples
+
+# stupid files
+yarn.lock
+package-lock.json

index.js

@@ -50,9 +50,10 @@ function Yuno (opts, cb) {
   var indexOpts = _.defaults(opts, {
     indexPath: this.indexPath,
     batchsize: 100,
+    fieldedSearch: false,
     nGramLength: 1,
-    separator: ' ',
-    stopwords: []
+    separator: /[|' .,\-|(\n)]+/,
+    stopwords: require('stopword').en
   })
 
   searchIndex(indexOpts, (err, si) => {

package.json

@@ -34,24 +34,25 @@
     "JSONStream": "^1.1.1",
     "batch-stream": "^0.1.3",
     "commander": "^2.9.0",
-    "jsonpath-plus": "^0.15.0",
+    "jsonpath-plus": "^0.16.0",
     "level": "^1.6.0",
     "level-batch-stream": "^1.3.1",
     "level-write-stream": "^1.0.0",
     "lodash": "^4.16.4",
     "mkdirp": "^0.5.1",
     "multi-write-stream": "^2.0.1",
     "natural": "blahah/natural",
-    "path-exists": "^2.1.0",
+    "path-exists": "^3.0.0",
     "pumpify": "^1.3.5",
     "search-index": "^0.13.0",
+    "stopword": "^0.1.6",
     "through2": "^2.0.1"
   },
   "devDependencies": {
-    "reuters-21578-json": "0.0.8",
     "end-of-stream": "^1.1.0",
+    "reuters-21578-json": "0.0.8",
     "rimraf": "^2.5.2",
-    "standard": "^6.0.8",
+    "standard": "^10.0.2",
     "stream-array": "^1.1.2",
     "tape": "^4.5.1",
     "temporary": "0.0.8"

preprocess/vector.js

@@ -21,7 +21,7 @@ function Vector (terms) {
 }
 
 function stripPunctuation (term) {
-  return term.replace(/\W+/g, '')
+  return term.replace(/\W+/g, '').replace(/<[^>]+>/g, '')
 }
 
 function stripTag (pair) {

test/search.js

@@ -42,7 +42,7 @@ test('streaming search', function (t) {
       var done = function (err) {
         t.error(err, 'search completes without error')
 
-        t.equals(results.length, 20, 'correct number of hits')
+        t.equals(results.length, 55, 'correct number of hits')
         rimraf(dbpath, {}, t.end)
       }