feat(datapackage): check datapackage actuality

Closes #544

doc/user-guide.md

@@ -11,6 +11,7 @@ Commands:
 Options:
   -v                     Print current version
   -i                     Generate datapackage.json
+  -a                     Check datapackage.json file actuality
   --compress-datapackage Compress datapackage.json file
   --translations         Rewrite "translations" section in existing datapackage.json
   --content              Rewrite "resources" and "ddfSchema" sections in existing datapackage.json
@@ -34,6 +35,7 @@ Examples:
   validate-ddf ../ddf-example -i                                                     generate datapackage.json file
   validate-ddf ../ddf-example -i --translations                                      update only "translations" section in datapackage.json
   validate-ddf ../ddf-example -i --translations --content                            rewrite "translations", "resources" and "ddfSchema" sections in datapackage.json
+  validate-ddf ../ddf-example -a                                                     check if datapackage.json in ../ddf-example folder actual
   validate-ddf ../ddf-example -j                                                     fix JSONs for this DDF dataset
   validate-ddf  --rules                                                              print information regarding supported rules
   validate-ddf ../ddf-example --summary      

src/cli-logic.ts

@@ -3,7 +3,7 @@ import { logger, settings, ddfRootFolder } from './utils';
 import { validationTransport } from './utils/logger';
 import { getRulesInformation } from './ddf-rules/registry';
 import { DdfJsonCorrector } from './ddf-definitions/ddf-json-corrector';
-import { StreamValidator, JSONValidator, validate, createDataPackage } from './index';
+import { StreamValidator, JSONValidator, validate, createDataPackage, getDataPackageActuality } from './index';
 import { checkLatestVersion } from './version';
 import { IssueView } from './ddf-rules/issue';
 
@@ -21,6 +21,16 @@ if (settings.heap) {
   v8.setFlagsFromString(`--max-old-space-size=${settings.heap}`);
 }
 
+if (settings.isDataPackageActual && !settings.versionShouldBePrinted) {
+  isValidationExpected = false;
+
+  getDataPackageActuality({ddfRootFolder}, (message) => {
+    console.log(message);
+
+    process.exit(0);
+  });
+}
+
 if (settings.isDataPackageGenerationMode && !settings.versionShouldBePrinted) {
   createDataPackage({ddfRootFolder}, (message) => {
     console.log(message);

src/data/db.ts

@@ -1,4 +1,4 @@
-import {createReadStream} from 'fs';
+import { createReadStream } from 'fs';
 
 const csv = require('fast-csv');
 
@@ -20,6 +20,7 @@ export class Db {
     let header = [];
     let ddfRecord = {};
     let lineNumber = 1;
+    let isError = false;
 
     fileStream.on('error', error => {
       onCollectionReady(error, []);
@@ -39,7 +40,15 @@ export class Db {
         ddfRecord = {};
         lineNumber++;
       })
-      .on('end', () => onCollectionReady(null, header));
+      .on('end', () => {
+        onCollectionReady(null, header);
+      })
+      .on('error', err => {
+        if (!isError) {
+          onCollectionReady(err);
+          isError = true;
+        }
+      });
   }
 
   getCollection(collectionName) {

src/ddf-definitions/ddf-data-set.ts

@@ -268,13 +268,15 @@ export class DdfDataSet {
     }
 
     const resources = this.getDataPackageResources().map(resource => {
-      if (!isArray(resource.schema.primaryKey)) {
-        resource.schema.primaryKey = [resource.schema.primaryKey];
+      const simplifiedResource = cloneDeep(resource);
+
+      if (!isArray(simplifiedResource.schema.primaryKey)) {
+        simplifiedResource.schema.primaryKey = [simplifiedResource.schema.primaryKey];
       }
 
-      resource.schema.fields = resource.schema.fields.map(field => field.name);
+      simplifiedResource.schema.fields = simplifiedResource.schema.fields.map(field => field.name);
 
-      return resource;
+      return simplifiedResource;
     });
 
     const conceptsResources = resources.filter(resource => getTypeByPrimaryKey(resource.schema.primaryKey) === CONCEPT);

src/index.ts

@@ -12,6 +12,7 @@ import { logger, getTransport, settings } from './utils';
 import * as fs from 'fs';
 import { DataPackage, DATA_PACKAGE_FILE } from './data/data-package';
 import { allRules } from './ddf-rules';
+import { difference, includes, isEmpty } from 'lodash';
 
 const child_process = require('child_process');
 const os = require('os');
@@ -347,6 +348,40 @@ export function createDataPackage(parameters: IDataPackageCreationParameters,
   });
 }
 
+export function getDataPackageActuality(parameters: IDataPackageCreationParameters, cb: Function) {
+  let {ddfPath, dataPackagePath, exists} = getDataPackageInfo(parameters.ddfRootFolder);
+
+  if (!exists) {
+    return cb('datapackage does not exist!');
+  }
+
+  let dataPackageContent = null;
+
+  try {
+    dataPackageContent = JSON.parse(fs.readFileSync(dataPackagePath, 'utf-8'));
+  } catch (err) {
+    cb(err.toString());
+  }
+
+
+  const dataPackage = new DataPackage(ddfPath, {silent: true});
+
+  dataPackage.take(() => {
+    const newDataPackageContent = dataPackage.getDataPackageObject();
+    const generatedResources = newDataPackageContent.resources.map(r => r.name);
+    const currentResources = dataPackageContent.resources.map(r => r.name);
+    const diff = difference(generatedResources, currentResources);
+
+    const newResources = diff.filter(r => includes(generatedResources, r));
+
+    if (!isEmpty(newResources)) {
+      cb(`datapackage is not actual: more details:\n${JSON.stringify(newResources, null, 2)}`);
+    } else {
+      cb(`\ndatapackage is actual\n`);
+    }
+  }, true);
+}
+
 export const validate = validator => {
   supervisor.abandon = false;
   resetGlobals();

src/utils/args.ts

@@ -13,6 +13,7 @@ const argv = yargs
   .example(`${myName} ../ddf-example --summary`, 'show summary data regarding the issues after validation')
   .example(`${myName} ../ddf-example -i --translations`, 'update only "translations" section in datapackage.json')
   .example(`${myName} ../ddf-example -i --translations --content`, 'rewrite "translations", "resources" and "ddfSchema" sections in datapackage.json')
+  .example(`${myName} ../ddf-example -a`, 'check if datapackage.json in ../ddf-example folder actual')
   .example(`${myName} ../ddf-example -j`, 'fix JSONs for this DDF dataset')
   .example(`${myName} --rules`, 'print information regarding supported rules')
   .example(`${myName} ../ddf-example --ws`, 'apply Waffle Server specific rules. See WAFFLE_SERVER_TAG tag based rules in the rules list (--rules flag)')
@@ -33,6 +34,7 @@ const argv = yargs
     'validate "ddf-example" and its subdirectories that contain spaces: case 2')
   .example(`${myName} ../ddf-example -i --compress-datapackage --heap 4096`, 'Create compressed datapackage.json via 4Gb heap')
   .describe('i', 'Generate datapackage.json file')
+  .describe('a', 'Check datapackage.json file actuality')
   .describe('compress-datapackage', 'Compress datapackage.json file')
   .describe('translations', 'Rewrite "translations" section in existing datapackage.json')
   .describe('content', 'Rewrite "resources" and "ddfSchema" sections in existing datapackage.json')
@@ -68,6 +70,7 @@ export const getSettings = () => {
   const options = ['include-tags', 'exclude-tags', 'include-rules', 'exclude-rules', 'exclude-dirs', 'heap'];
   const setMiscSettings = () => {
     settings.isDataPackageGenerationMode = !!argv.i;
+    settings.isDataPackageActual = !!argv.a;
     settings.isJsonAutoCorrectionMode = !!argv.j;
     settings.versionShouldBePrinted = !!argv.v;
     settings.datapointlessMode = !!argv.datapointless;

test/e2e.spec.ts

@@ -4,37 +4,60 @@ import { exec } from 'child_process';
 
 const expect = chai.expect;
 
-function execute(command, callback){
-  exec(command, function(error, stdout, stderr) {
+function execute(command, callback) {
+  exec(command, function (error, stdout, stderr) {
     callback(error, stdout, stderr);
   });
 }
 
 describe('e2e', () => {
-  it('an issue should be found when "ws" flag is ON and DS contains WS based issue', done => {
-    execute('./src/cli.js ./test/fixtures/rules-cases/entity-value-as-entity-name --silent --ws', (error, stdout, stderr) => {
-      expect(!!error).to.be.true;
-      expect(error.code).to.be.equal(1);
-      expect(!!stdout).to.be.true;
+  describe('--ws flag', () => {
+    it('any issue should NOT be found when "--ws" flag is OFF and DS contains WS based issue', done => {
+      execute('./src/cli.js ./test/fixtures/rules-cases/entity-value-as-entity-name --silent', (error, stdout, stderr) => {
+        expect(!!error).to.be.false;
+        expect(!!stdout).to.be.false;
+        expect(!!stderr).to.be.false;
+
+        done();
+      });
+    });
+
+    it('an issue should be found when "--ws" flag is ON and DS contains WS based issue', done => {
+      execute('./src/cli.js ./test/fixtures/rules-cases/entity-value-as-entity-name --silent --ws', (error, stdout, stderr) => {
+        expect(!!error).to.be.true;
+        expect(error.code).to.be.equal(1);
+        expect(!!stdout).to.be.true;
 
-      const issues = JSON.parse(stdout);
-      const issue: {id: string} = <{id: string}>head(issues);
+        const issues = JSON.parse(stdout);
+        const issue: { id: string } = <{ id: string }>head(issues);
 
-      expect(issues.length).to.be.equals(1);
-      expect(issue.id).to.be.equals('ENTITY_VALUE_AS_ENTITY_NAME');
-      expect(!!stderr).to.be.false;
+        expect(issues.length).to.be.equals(1);
+        expect(issue.id).to.be.equals('ENTITY_VALUE_AS_ENTITY_NAME');
+        expect(!!stderr).to.be.false;
 
-      done();
+        done();
+      });
     });
   });
 
-  it('any issue should NOT be found when "ws" flag is OFF and DS contains WS based issue', done => {
-    execute('./src/cli.js ./test/fixtures/rules-cases/entity-value-as-entity-name --silent', (error, stdout, stderr) => {
-      expect(!!error).to.be.false;
-      expect(!!stdout).to.be.false;
-      expect(!!stderr).to.be.false;
+  describe('-a flag', () => {
+    it('any issue should NOT be found when "-a" flag is OFF and DS contains WS based issue', done => {
+      execute('./src/cli.js ./test/fixtures/good-folder-dp --silent', (error, stdout, stderr) => {
+        expect(!!error).to.be.false;
+        expect(!!stdout).to.be.false;
+        expect(!!stderr).to.be.false;
+
+        done();
+      });
+    });
+
+    it('an issue should be found when "-a" flag is ON and DS contains WS based issue', done => {
+      execute('./src/cli.js ./test/fixtures/outdated-datapackage --silent -a', (error, stdout, stderr) => {
+        expect(!!error).to.be.false;
+        expect(stdout).to.be.equal("datapackage is not actual: more details:\n[\n  \"gas_production_bcf--by--geo--year\"\n]\n");
 
-      done();
+        done();
+      });
     });
   });
 });

test/fixtures/outdated-datapackage/datapackage.json

@@ -0,0 +1,83 @@
+{
+  "name": "outdated-datapackage",
+  "title": "outdated-datapackage",
+  "description": "",
+  "version": "0.0.1",
+  "language": {
+    "id": "en",
+    "name": "English"
+  },
+  "translations": [],
+  "license": "",
+  "author": "",
+  "resources": [
+    {
+      "path": "ddf--concepts.csv",
+      "name": "ddf--concepts",
+      "schema": {
+        "fields": [
+          {
+            "name": "concept"
+          },
+          {
+            "name": "concept_type"
+          },
+          {
+            "name": "name"
+          }
+        ],
+        "primaryKey": "concept"
+      }
+    },
+    {
+      "path": "ddf--entities--geo.csv",
+      "name": "geo",
+      "schema": {
+        "fields": [
+          {
+            "name": "geo"
+          },
+          {
+            "name": "geo_name"
+          }
+        ],
+        "primaryKey": "geo"
+      }
+    }
+  ],
+  "ddfSchema": {
+    "datapoints": [],
+    "entities": [
+      {
+        "primaryKey": [
+          "geo"
+        ],
+        "value": "geo_name",
+        "resources": [
+          "geo"
+        ]
+      }
+    ],
+    "concepts": [
+      {
+        "primaryKey": [
+          "concept"
+        ],
+        "value": "concept_type",
+        "resources": [
+          "ddf--concepts"
+        ]
+      },
+      {
+        "primaryKey": [
+          "concept"
+        ],
+        "value": "name",
+        "resources": [
+          "ddf--concepts"
+        ]
+      }
+    ],
+    "synonyms": []
+  }
+}

test/fixtures/outdated-datapackage/ddf--concepts.csv

@@ -0,0 +1,6 @@
+concept,concept_type,name
+geo,entity_domain,Geo
+gas_production_bcf,measure,Gas Production – Bcf
+name,string,Name
+geo_name,string,Name
+year,time,Year