feat(rules): WRONG_INDEX_VALUE and WRONG_INDEX_KEY implementation

Closes #70 #71

doc/rules/WRONG_INDEX_KEY.md

@@ -0,0 +1,52 @@
+# WRONG_INDEX_KEY
+
+## Rule test folder 
+
+`test/fixtures/rules-cases/wrong-index-key`
+
+## Description
+An issue according to this rule will be fired when `key` in `ddf--index.csv` is not valid. The key should be concept or set of concepts or `concept` constant.
+
+## Examples of correct data
+
+`ddf--index.csv`
+```
+key,value,file
+concept,concept_type,ddf--concepts.csv
+concept,name,ddf--concepts.csv
+"geo,year",gas_production_bcf,ddf--datapoints--gas_production_bcf--by--geo--year.csv
+geo,geo_name,ddf--entities--geo.csv
+```
+
+`ddf--concepts.csv`
+```
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+year,time,Year
+```
+
+## Examples of incorrect data
+
+`ddf--index.csv`
+```
+key,value,file
+concept,concept_type,ddf--concepts.csv
+concept,name,ddf--concepts.csv
+"geo,foo",gas_production_bcf,ddf--datapoints--gas_production_bcf--by--geo--year.csv
+bar,geo_name,ddf--entities--geo.csv
+```
+
+`ddf--concepts.csv`
+```
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+year,time,Year
+```
+
+## Output data format
+
+Non existing concepts that found in record's key of DDF index

doc/rules/WRONG_INDEX_VALUE.md

@@ -0,0 +1,51 @@
+# WRONG_INDEX_VALUE
+
+## Rule test folder 
+
+`test/fixtures/rules-cases/wrong-index-value`
+
+## Description
+An issue according to this rule will be fired when `value` in `ddf--index.csv` is not valid. The key should be concept or set of concepts or `concept` constant.
+
+## Examples of correct data
+
+`ddf--index.csv`
+```
+key,value,file
+concept,concept_type,ddf--concepts.csv
+concept,name,ddf--concepts.csv
+"geo,year",gas_production_bcf,ddf--datapoints--gas_production_bcf--by--geo--year.csv
+geo,geo_name,ddf--entities--geo.csv
+```
+`ddf--concepts.csv`
+```
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+geo_name,string,Name
+year,time,Year
+```
+
+* **Examples of incorrect data:**
+
+`ddf--index.csv`
+```
+key,value,file
+concept,concept_type,ddf--concepts.csv
+concept,name,ddf--concepts.csv
+"geo,year",foo,ddf--datapoints--gas_production_bcf--by--geo--year.csv
+geo,geo_name,ddf--entities--geo.csv
+```
+`ddf--concepts.csv`
+```
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+year,time,Year
+```
+
+## Output data format
+
+Non existing concepts that found in record's value of DDF index

lib/data/ddf-index.js

@@ -81,7 +81,7 @@ class DdfIndex {
 
   /*eslint max-statements: [2, 20]*/
   fillType(fileDescriptor) {
-    if (fileDescriptor.type !== null) {
+    if (fileDescriptor.type !== null || _.isEmpty(fileDescriptor.headers)) {
       return;
     }
 

lib/data/file-descriptor.js

@@ -50,7 +50,7 @@ class FileDescriptor {
         return;
       }
 
-      // todo: do it more correctly
+      // todo: process separator
       this.headers = line.split(',');
 
       cb();

lib/ddf-rules/index-rules.js

@@ -4,6 +4,37 @@ const _ = require('lodash');
 const registry = require('./registry');
 const Issue = require('./issue');
 
+const CONCEPT_TERMS = ['concept', 'concept_type'];
+
+function getWrongIndexKeyOrValueIssues(kind, rule) {
+  return (ddfDataSet, ddfIndex) => {
+    function getUndefinedConcept(concept) {
+      const foundConcept = ddfDataSet.getConcept().getRecordByKey(concept);
+
+      return !!foundConcept || _.includes(CONCEPT_TERMS, concept) ? null : concept;
+    }
+
+    const wrongKeys = _.compact(_.flatten(
+      ddfIndex.content.map(indexRecord => {
+        const keys = indexRecord[kind].split(',');
+
+        return _.compact(keys.map(key => getUndefinedConcept(key)));
+      })
+    ));
+
+    if (_.isEmpty(wrongKeys)) {
+      return null;
+    }
+
+    return new Issue(rule)
+      .setPath(ddfIndex.ddfPath)
+      .setData(wrongKeys);
+  };
+}
+
+const getWrongIndexKeyIssues = getWrongIndexKeyOrValueIssues('key', registry.WRONG_INDEX_KEY);
+const getWrongIndexValueIssues = getWrongIndexKeyOrValueIssues('value', registry.WRONG_INDEX_VALUE);
+
 module.exports = {
   [registry.INCORRECT_FILE]: ddfDataSet => _.flattenDeep(
     ddfDataSet.ddfRoot.directoryDescriptors
@@ -17,5 +48,15 @@ module.exports = {
   [registry.INDEX_IS_NOT_FOUND]: ddfDataSet => ddfDataSet.ddfRoot.directoryDescriptors
     .filter(directoryDescriptor => directoryDescriptor.isDDF && !!directoryDescriptor.ddfIndex.error)
     .map(directoryDescriptor => new Issue(registry.INDEX_IS_NOT_FOUND)
-      .setPath(directoryDescriptor.dir))
+      .setPath(directoryDescriptor.dir)),
+  [registry.WRONG_INDEX_KEY]: ddfDataSet => ddfDataSet.ddfRoot.directoryDescriptors
+    .filter(directoryDescriptor => directoryDescriptor.isDDF)
+    .map(directoryDescriptor =>
+      getWrongIndexKeyIssues(ddfDataSet, directoryDescriptor.ddfIndex))
+    .filter(issue => !!issue),
+  [registry.WRONG_INDEX_VALUE]: ddfDataSet => ddfDataSet.ddfRoot.directoryDescriptors
+    .filter(directoryDescriptor => directoryDescriptor.isDDF)
+    .map(directoryDescriptor =>
+      getWrongIndexValueIssues(ddfDataSet, directoryDescriptor.ddfIndex))
+    .filter(issue => !!issue)
 };

lib/ddf-rules/registry.js

@@ -18,6 +18,8 @@ exports.WRONG_ENTITY_IS_VALUE = Symbol.for('WRONG_ENTITY_IS_VALUE');
 exports.FILENAME_DOES_NOT_MATCH_HEADER = Symbol.for('FILENAME_DOES_NOT_MATCH_HEADER');
 exports.CONCEPT_MANDATORY_FIELD_NOT_FOUND = Symbol.for('CONCEPT_MANDATORY_FIELD_NOT_FOUND');
 exports.CONCEPTS_NOT_FOUND = Symbol.for('CONCEPTS_NOT_FOUND');
+exports.WRONG_INDEX_KEY = Symbol.for('WRONG_INDEX_KEY');
+exports.WRONG_INDEX_VALUE = Symbol.for('WRONG_INDEX_VALUE');
 
 exports.WARNING_TAG = Symbol.for('WARNING');
 exports.WAFFLE_SERVER_TAG = Symbol.for('WAFFLE_SERVER');
@@ -45,7 +47,9 @@ exports.tags = {
   [exports.WRONG_ENTITY_IS_VALUE]: [],
   [exports.FILENAME_DOES_NOT_MATCH_HEADER]: [exports.WAFFLE_SERVER_TAG],
   [exports.CONCEPT_MANDATORY_FIELD_NOT_FOUND]: [exports.WAFFLE_SERVER_TAG],
-  [exports.CONCEPTS_NOT_FOUND]: [exports.WAFFLE_SERVER_TAG]
+  [exports.CONCEPTS_NOT_FOUND]: [exports.WAFFLE_SERVER_TAG],
+  [exports.WRONG_INDEX_KEY]: [],
+  [exports.WRONG_INDEX_VALUE]: []
 };
 
 exports.descriptions = {
@@ -66,7 +70,9 @@ exports.descriptions = {
   [exports.WRONG_ENTITY_IS_VALUE]: 'Wrong value for "is" header',
   [exports.FILENAME_DOES_NOT_MATCH_HEADER]: 'Filename does not match to header of this file',
   [exports.CONCEPT_MANDATORY_FIELD_NOT_FOUND]: 'Concept mandatory field is not found',
-  [exports.CONCEPTS_NOT_FOUND]: 'Concepts are not found'
+  [exports.CONCEPTS_NOT_FOUND]: 'Concepts are not found',
+  [exports.WRONG_INDEX_KEY]: 'Wrong Index key',
+  [exports.WRONG_INDEX_VALUE]: 'Wrong Index value'
 };
 
 exports.getRulesInformation = () => Object.getOwnPropertySymbols(exports.descriptions)

lib/utils/file.js

@@ -107,22 +107,22 @@ function getFileLine(filename, lineNo, callback) {
 
   let fileData = '';
 
-  stream.on('data', function (data) {
+  stream.on('data', data => {
     fileData += data;
 
-    const lines = fileData.split("\n");
+    const lines = fileData.split('\n');
 
     if (lines.length >= +lineNo) {
       stream.destroy();
       callback(null, lines[+lineNo]);
     }
   });
 
-  stream.on('error', function () {
-    callback('Error', null);
+  stream.on('error', err => {
+    callback(err, null);
   });
 
-  stream.on('end', function () {
+  stream.on('end', () => {
     callback('File end reached without finding line', null);
   });
 }

test/ddf-index-rules.spec.js

@@ -1,4 +1,5 @@
 'use strict';
+const _ = require('lodash');
 const chai = require('chai');
 const sinonChai = require('sinon-chai');
 const DdfDataSet = require('../lib/ddf-definitions/ddf-data-set');
@@ -114,4 +115,79 @@ describe('rules for index', () => {
       });
     });
   });
+
+  describe('when index based rules checking', () => {
+    afterEach(done => {
+      ddfDataSet.dismiss(() => {
+        done();
+      });
+    });
+
+    describe('and WRONG_INDEX_KEY rule', () => {
+      it('any issue should NOT be found for good folder', done => {
+        ddfDataSet = new DdfDataSet('./test/fixtures/good-folder-indexed');
+
+        ddfDataSet.load(() => {
+          const results = indexRules[rulesRegistry.WRONG_INDEX_KEY](ddfDataSet);
+
+          expect(_.isEmpty(results)).to.be.true;
+
+          done();
+        });
+      });
+
+      it(`expected issue should be found for folder with the problem
+    (fixtures/rules-cases/wrong-index-key)`, done => {
+        ddfDataSet = new DdfDataSet('./test/fixtures/rules-cases/wrong-index-key');
+
+        ddfDataSet.load(() => {
+          const results = indexRules[rulesRegistry.WRONG_INDEX_KEY](ddfDataSet);
+
+          expect(results.length).to.equal(1);
+
+          const result = _.head(results);
+          const expectedIssueData = ['foo', 'bar'];
+
+          expect(result.type).to.equal(rulesRegistry.WRONG_INDEX_KEY);
+          expect(_.isEqual(result.data, expectedIssueData)).to.be.true;
+
+          done();
+        });
+      });
+    });
+
+    describe('and WRONG_INDEX_VALUE rule', () => {
+      it('any issue should NOT be found for good folder', done => {
+        ddfDataSet = new DdfDataSet('./test/fixtures/good-folder-indexed');
+
+        ddfDataSet.load(() => {
+          const results = indexRules[rulesRegistry.WRONG_INDEX_VALUE](ddfDataSet);
+
+          expect(_.isEmpty(results)).to.be.true;
+
+          done();
+        });
+      });
+
+      it(`expected issue should be found for folder with the problem
+    (fixtures/rules-cases/wrong-index-value)`, done => {
+        ddfDataSet = new DdfDataSet('./test/fixtures/rules-cases/wrong-index-value');
+
+        ddfDataSet.load(() => {
+          const results = indexRules[rulesRegistry.WRONG_INDEX_VALUE](ddfDataSet);
+
+          expect(results.length).to.equal(1);
+
+          const result = _.head(results);
+
+          const expectedIssueData = ['foo', 'geo_name'];
+
+          expect(result.type).to.equal(rulesRegistry.WRONG_INDEX_VALUE);
+          expect(_.isEqual(result.data, expectedIssueData)).to.be.true;
+
+          done();
+        });
+      });
+    });
+  });
 });

test/fixtures/good-folder-indexed/ddf--concepts.csv

@@ -0,0 +1,6 @@
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+geo_name,string,Name
+year,time,Year