Do not index single chars

This only pollutes the record, since we never do searches on a single character.
DeuxHuitHuit · Jan 28, 2019 · 099c8ba · 099c8ba
1 parent 85d4c57
commit 099c8ba
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 7 deletions.
diff --git a/lib/process.js b/lib/process.js
@@ -15,6 +15,7 @@ const trim = (s) => !s ? null : _trim(s);
 const entities = (H => new H.XmlEntities)(require('html-entities'));
 const defaultAttributes = ['content', 'value'];
 const defaultPoppedAttributes = ['text'];
+const minStringLength = 1;
 
 const recursiveFindValue = (node, array, attribs) => {
 	if (!node ||  node.type === 'comment') {
@@ -23,8 +24,8 @@ const recursiveFindValue = (node, array, attribs) => {
 
 	// First, if we found a text node, use its value
 	if (node.type === 'text' || !!node.data) {
-		let text = trim(node.data);
-		if (!!text) {
+		const text = trim(node.data);
+		if (!!text && text.length > minStringLength) {
 			array.push(entities.decode(text));
 			// No need to check further
 			return array;

diff --git a/test/parse.js b/test/parse.js
@@ -110,11 +110,11 @@ test('Parse with limit', (t) => {
 	};
 	const c = _.cloneDeep(config);
 	const data = `<html><head>
-	<title>1</title>
-	<title>2</title>
-	<title>3</title>
-	<title>4</title>
-	<title>5</title>
+	<title>test-1</title>
+	<title>test-2</title>
+	<title>test-3</title>
+	<title>test-4</title>
+	<title>test-5</title>
 <head></html>`;
 	parse(rec, data, c);
 	t.equal(rec.title.length, 5);
@@ -126,3 +126,23 @@ test('Parse with limit', (t) => {
 	t.equal(rec.title.length, 3);
 	t.end();
 });
+
+test('Parse no min char', (t) => {
+	const rec = {
+		date: now,
+		timestamp: now.getTime()
+	};
+	const c = _.cloneDeep(config);
+	const data = `<html><head>
+	<title>test</title>
+	<title>t</title>
+	<title>test2</title>
+<head></html>`;
+	parse(rec, data, c);
+	t.equal(rec.date, now);
+	t.equal(rec.timestamp, now.getTime());
+	t.equal(rec.title.length, 2);
+	t.equal(rec.title[0], 'test');
+	t.equal(rec.title[1], 'test2');
+	t.end();
+});