Skip to content

Commit 29a8af9

Browse files
buchslavabuchslava
committed
feat(general): show current progress/file during validation
Closes #142
1 parent 616cda7 commit 29a8af9

File tree

12 files changed

+242
-91
lines changed

12 files changed

+242
-91
lines changed

‎package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"test-travis": "istanbul cover mocha _mocha -- -R spec --timeout 200000 --compilers ts:ts-node/register --recursive test/**/*.spec.ts && codecov",
1313
"changelog": "conventional-changelog -i CHANGELOG.md -s -p angular",
1414
"github-release": "conventional-github-releaser -p angular",
15-
"build": "tsc && touch lib/package.json && echo \\{\\\"version\\\": \\\"1.7.6\\\"\\} > lib/package.json",
15+
"build": "tsc && touch lib/package.json && echo \\{\\\"version\\\": \\\"1.8.0\\\"\\} > lib/package.json",
1616
"prepublish": "npm run build",
1717
"preversion": "npm test",
1818
"version": "npm run changelog && git add CHANGELOG.md",
@@ -44,7 +44,7 @@
4444
"deep-diff": "0.3.4",
4545
"fast-csv": "2.4.0",
4646
"fs": "0.0.2",
47-
"jsonexport": "^1.5.2",
47+
"jsonexport": "1.5.2",
4848
"levenshtein": "1.0.5",
4949
"lodash": "4.17.4",
5050
"md5": "2.2.1",

‎src/cli-logic.ts

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as fs from 'fs';
33
import * as path from 'path';
44
import { isString } from 'lodash';
55
import { logger, settings, ddfRootFolder } from './utils';
6+
import { validationTransport } from './utils/logger';
67
import { getRulesInformation } from './ddf-rules/registry';
78
import { DataPackage } from './data/data-package';
89
import { DdfJsonCorrector } from './ddf-definitions/ddf-json-corrector';
@@ -89,9 +90,11 @@ if (!isValidationExpected) {
8990
if (isValidationExpected) {
9091
const validator = new StreamValidator(ddfRootFolder, settings);
9192

92-
logger.notice('[');
93+
let hasIssue = false;
9394

9495
validator.on('issue', (issue: any) => {
96+
hasIssue = true;
97+
9598
if (isString(issue)) {
9699
logger.notice(issue + '\n');
97100
}
@@ -106,7 +109,13 @@ if (isValidationExpected) {
106109
throw err;
107110
}
108111

109-
logger.notice('{}]\n');
112+
if (settings.progress && hasIssue) {
113+
console.log(`\nValidation was finished with issues. Details are here: ${validationTransport.file}.`);
114+
}
115+
116+
if (settings.progress && !hasIssue) {
117+
console.log('\nValidation was finished successfully.');
118+
}
110119

111120
checkLatestVersion(localPackage.version);
112121
});

‎src/data/csv-checker.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ const getErrors = parsedCsv => parsedCsv.errors
1414
export class CsvChecker {
1515
public filePath: string;
1616
public error: any;
17-
public errors: Array<any> = [];
17+
public errors: any[] = [];
1818

1919
constructor(filePath) {
2020
this.filePath = filePath;

‎src/data/data-package.ts

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ export class DataPackage {
126126
public errors: any[];
127127
public warnings: any[];
128128
public fileDescriptors: IDdfFileDescriptor[];
129+
public fileDescriptorsHash: Map<string, IDdfFileDescriptor>;
129130
public dataPackageContent: any;
130131
public translationFolders: any[];
131132
public db: Db;
@@ -328,8 +329,7 @@ export class DataPackage {
328329
path: `${getRelativeDir(fileDescriptor.fullPath)}${fileDescriptor.filename}`,
329330
name: `${stripDdfPrefix(fileDescriptor.name)}${getNameSuffix(fileDescriptor.directoryIndex)}`,
330331
schema: {
331-
fields: (fileDescriptor.headers || [])
332-
.map(header => prepareField(header, fileDescriptor)),
332+
fields: (fileDescriptor.headers || []).map(header => prepareField(header, fileDescriptor)),
333333
primaryKey: fileDescriptor.primaryKey
334334
}
335335
}))
@@ -339,11 +339,15 @@ export class DataPackage {
339339
getType(filename) {
340340
const normalizedFileName = path.resolve(this.rootFolder, filename);
341341

342-
return head(
343-
this.fileDescriptors
344-
.filter(fileDescriptor => path.resolve(this.rootFolder, fileDescriptor.fullPath) === normalizedFileName)
345-
.map(fileDescriptor => fileDescriptor.type)
346-
);
342+
if (!this.fileDescriptorsHash) {
343+
this.fileDescriptorsHash = new Map<string, IDdfFileDescriptor>();
344+
345+
this.fileDescriptors.forEach(fileDescriptor => {
346+
this.fileDescriptorsHash.set(path.resolve(this.rootFolder, fileDescriptor.fullPath), fileDescriptor);
347+
});
348+
}
349+
350+
return this.fileDescriptorsHash.get(normalizedFileName).type;
347351
}
348352

349353
build(onDataPackageReady) {

‎src/data/ddf-root.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { resolve } from 'path';
55
import { TRNSLATIONS_FOLDER } from '../ddf-definitions/constants';
66
import { DataPackage } from './data-package';
77
import { FileDescriptor } from './file-descriptor';
8+
import { logger } from '../utils';
89

910
const PROCESS_LIMIT = 30;
1011

@@ -75,13 +76,13 @@ export class DDFRoot {
7576

7677
this.fileDescriptors = expectedResources.map(ddfResource => this.getFileDescriptor(this.path, ddfResource));
7778

78-
const actionsCsv = this.fileDescriptors.map(fileDescriptor => onFileChecked =>
79-
fileDescriptor.csvChecker.check(onFileChecked));
80-
const actionsForDescriptor = this.fileDescriptors.map(fileDescriptor =>
79+
logger.progressInit('root checking', {total: this.fileDescriptors.length});
80+
81+
const actions = this.fileDescriptors.map(fileDescriptor =>
8182
onFileChecked => fileDescriptor.check(onFileChecked));
8283

8384
parallelLimit(
84-
actionsCsv.concat(actionsForDescriptor),
85+
actions,
8586
PROCESS_LIMIT,
8687
checkErr => {
8788
if (checkErr) {

‎src/data/file-descriptor.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { stat } from 'fs';
44
import { getFileLine } from '../utils/file';
55
import { INCORRECT_FILE } from '../ddf-rules/registry';
66
import { CsvChecker } from './csv-checker';
7+
import { logger } from '../utils';
78

89
const PROCESS_LIMIT = 30;
910

@@ -102,11 +103,16 @@ export class FileDescriptor {
102103
}
103104

104105
if (isEmpty(this.issues)) {
105-
this.csvChecker.check(() => onFileDescriptorChecked());
106+
this.csvChecker.check(() => {
107+
logger.progress();
108+
onFileDescriptorChecked();
109+
});
106110

107111
return;
108112
}
109113

114+
logger.progress();
115+
110116
onFileDescriptorChecked(this.issues);
111117
});
112118
});

‎src/ddf-definitions/ddf-data-set.ts

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { DataPoint } from './data-point';
77
import { Db } from '../data/db';
88
import { DDFRoot } from '../data/ddf-root';
99
import { DataPackage } from '../data/data-package';
10+
import { logger } from '../utils';
1011

1112
export class DdfDataSet {
1213
public db: Db;
@@ -30,33 +31,39 @@ export class DdfDataSet {
3031
};
3132

3233
const processFileDescriptors = (ddfRoot: DDFRoot) => {
33-
ddfRoot.fileDescriptors
34-
.filter(fileDescriptor => isEmpty(fileDescriptor.issues))
35-
.forEach(fileDescriptor => {
36-
if (fileDescriptor.is(DATA_POINT)) {
37-
loaders.push(onFileLoaded => {
38-
fileDescriptor.fillHeaders(() => {
39-
this.expectedClass[fileDescriptor.type].addDescriptors(fileDescriptor, ddfRoot.dataPackageDescriptor);
40-
onFileLoaded();
41-
});
34+
const expectedFileDescriptors = ddfRoot.fileDescriptors.filter(fileDescriptor => isEmpty(fileDescriptor.issues));
35+
36+
logger.progressInit('dataset loading', {total: expectedFileDescriptors.length});
37+
38+
expectedFileDescriptors.forEach(fileDescriptor => {
39+
if (fileDescriptor.is(DATA_POINT)) {
40+
loaders.push(onFileLoaded => {
41+
fileDescriptor.fillHeaders(() => {
42+
this.expectedClass[fileDescriptor.type].addDescriptors(fileDescriptor, ddfRoot.dataPackageDescriptor);
43+
44+
logger.progress();
45+
46+
onFileLoaded();
4247
});
43-
}
44-
45-
if (fileDescriptor.is([CONCEPT, ENTITY])) {
46-
loaders.push(onFileLoaded => {
47-
fileDescriptor.fillHeaders(() => {
48-
this.db.fillCollection(
49-
Symbol.keyFor(fileDescriptor.type),
50-
fileDescriptor.fullPath,
51-
fileErr => {
52-
this.expectedClass[fileDescriptor.type].addFileDescriptor(fileDescriptor);
53-
this.expectedClass[fileDescriptor.type].getTranslationsData(translationsErr =>
54-
onFileLoaded(fileErr || translationsErr));
55-
}, false);
56-
});
48+
});
49+
}
50+
51+
if (fileDescriptor.is([CONCEPT, ENTITY])) {
52+
loaders.push(onFileLoaded => {
53+
fileDescriptor.fillHeaders(() => {
54+
this.db.fillCollection(
55+
Symbol.keyFor(fileDescriptor.type),
56+
fileDescriptor.fullPath,
57+
fileErr => {
58+
this.expectedClass[fileDescriptor.type].addFileDescriptor(fileDescriptor);
59+
this.expectedClass[fileDescriptor.type].getTranslationsData(translationsErr => onFileLoaded(fileErr || translationsErr));
60+
61+
logger.progress();
62+
}, false);
5763
});
58-
}
59-
});
64+
});
65+
}
66+
});
6067
};
6168

6269
if (this.ddfRoot.isDDF) {

‎src/index.ts

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import * as path from 'path';
22
import { EventEmitter } from 'events';
33
import { parallelLimit } from 'async';
4-
import { isEmpty, flattenDeep, concat, isArray } from 'lodash';
4+
import { isEmpty, flattenDeep, concat, isArray, compact } from 'lodash';
55
import { DdfDataSet } from './ddf-definitions/ddf-data-set';
66
import { allRules as ddfRules } from './ddf-rules';
77
import { IssuesFilter } from './utils/issues-filter';
@@ -17,6 +17,7 @@ import {
1717
getDataPointFileChunks,
1818
createRecordBasedRulesProcessor
1919
} from './shared';
20+
import { logger } from './utils';
2021

2122
const child_process = require('child_process');
2223
const os = require('os');
@@ -29,12 +30,21 @@ function clearGlobalStates() {
2930
}
3031

3132
function processSimpleRules(context, onIssue) {
32-
Object.getOwnPropertySymbols(ddfRules)
33-
.filter(key => isSimpleRule(ddfRules[key]))
34-
.filter(key => context.issuesFilter.isAllowed(key))
35-
.map(key => ddfRules[key].rule(context.ddfDataSet))
36-
.filter(issues => !isEmpty(issues))
37-
.forEach(issue => onIssue(issue));
33+
const rulesKeys = Object.getOwnPropertySymbols(ddfRules).filter(key => isSimpleRule(ddfRules[key]) && context.issuesFilter.isAllowed(key));
34+
35+
for (let key of rulesKeys) {
36+
const issues = ddfRules[key].rule(context.ddfDataSet);
37+
38+
if (!isEmpty(issues)) {
39+
if (isArray(issues)) {
40+
issues.forEach(issue => onIssue(issue));
41+
}
42+
43+
if (!isArray(issues)) {
44+
onIssue(issues)
45+
}
46+
}
47+
}
3848
}
3949

4050
export function createRecordBasedRuleProcessor(context, fileDescriptor, resultHandler) {
@@ -128,7 +138,7 @@ export class JSONValidator {
128138
childProcess.on('message', (message) => {
129139
childProcessesFinished++;
130140

131-
this.out = this.out.concat(message.out);
141+
this.out = compact(this.out.concat(message.out));
132142

133143
if (childProcessesFinished === cpuCount) {
134144
this.issueEmitter.emit('finish', message.err, this.out);
@@ -174,7 +184,9 @@ export class JSONValidator {
174184
}
175185

176186
if (!this.settings.isMultithread) {
177-
parallelLimit(getValidationActions(this), CONCURRENT_OPERATIONS_AMOUNT, err => {
187+
const actions = getValidationActions(this);
188+
189+
parallelLimit(actions, CONCURRENT_OPERATIONS_AMOUNT, err => {
178190
this.issueEmitter.emit('finish', err, this.out);
179191
});
180192
}
@@ -202,6 +214,8 @@ export class StreamValidator {
202214
if (!isEmpty(result)) {
203215
result.map(issue => this.issueEmitter.emit('issue', issue.view()));
204216
}
217+
}, () => {
218+
logger.progress();
205219
});
206220
}
207221

@@ -211,6 +225,9 @@ export class StreamValidator {
211225

212226
multiThreadProcessing() {
213227
const fileChunks = getDataPointFileChunks(this.ddfDataSet, cpuCount);
228+
const total = fileChunks.reduce((result, chunk) => result + chunk.length, 0);
229+
230+
logger.progressInit('datapoints validation', {total});
214231

215232
let childProcessesFinished = 0;
216233

@@ -222,6 +239,10 @@ export class StreamValidator {
222239
childProcessesFinished++;
223240
}
224241

242+
if (message.progress) {
243+
logger.progress();
244+
}
245+
225246
if (!message.finish && message.issue) {
226247
this.issueEmitter.emit('issue', message.issue);
227248
}
@@ -270,7 +291,11 @@ export class StreamValidator {
270291
}
271292

272293
if (!this.settings.isMultithread) {
273-
parallelLimit(getValidationActions(this), CONCURRENT_OPERATIONS_AMOUNT, err => {
294+
const actions = getValidationActions(this);
295+
296+
logger.progressInit('datapoints validation', {total: actions.length});
297+
298+
parallelLimit(actions, CONCURRENT_OPERATIONS_AMOUNT, err => {
274299
this.issueEmitter.emit('finish', err);
275300
});
276301
}

‎src/shared.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { compact, isArray, sortBy } from 'lodash';
1+
import { compact, isArray, sortBy, flattenDeep } from 'lodash';
22
import { allRules as ddfRules } from './ddf-rules';
33
import { FileDescriptor } from './data/file-descriptor';
44
import { DdfDataSet } from './ddf-definitions/ddf-data-set';
@@ -43,7 +43,13 @@ export const noTranslation = (key, fileDescriptor) => !fileDescriptor.isTranslat
4343
export const toArray = value => compact(isArray(value) ? value : [value]);
4444
export const getDataPointFileChunks = (ddfDataSet: DdfDataSet, cpuCount: number): string[] => {
4545
const fileChunks = [];
46-
const filesSortedBySize = sortBy(ddfDataSet.getDataPoint().fileDescriptors, ['size'])
46+
const expectedFileDescriptors = ddfDataSet.getDataPoint().fileDescriptors;
47+
const translationFileDescriptors = flattenDeep(ddfDataSet.getDataPoint().fileDescriptors.map(fileDescriptor =>
48+
fileDescriptor.getExistingTranslationDescriptors()));
49+
50+
expectedFileDescriptors.push(...translationFileDescriptors);
51+
52+
const filesSortedBySize = sortBy(expectedFileDescriptors, ['size'])
4753
.map((fileDescriptor: FileDescriptor) => fileDescriptor.fullPath);
4854

4955
for (let index = 0; index < cpuCount; index++) {
@@ -61,7 +67,7 @@ export const getDataPointFileChunks = (ddfDataSet: DdfDataSet, cpuCount: number)
6167
return fileChunks;
6268
};
6369

64-
export function createRecordBasedRulesProcessor(context, fileDescriptor, resultHandler) {
70+
export function createRecordBasedRulesProcessor(context, fileDescriptor, resultHandler, progressHandler?) {
6571
const ddfDataSet = context.ddfDataSet;
6672

6773
return onDataPointReady => {
@@ -70,6 +76,11 @@ export function createRecordBasedRulesProcessor(context, fileDescriptor, resultH
7076
createRecordAggregationProcessor(context, ddfDataSet, fileDescriptor, resultHandler),
7177
() => {
7278
processAggregation(context, ddfDataSet, fileDescriptor, resultHandler);
79+
80+
if (progressHandler) {
81+
progressHandler();
82+
}
83+
7384
onDataPointReady();
7485
}
7586
);

0 commit comments

Comments
 (0)