Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#I285: removed empty objects and arrays from sample data #261

Merged
merged 4 commits into from
Oct 24, 2024
Merged

#I285: removed empty objects and arrays from sample data #261

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
518ee54
#I285: removed empty objects and arrays from sample data
HarishGangula Oct 24, 2024
19513e5
Merge branch 'main' into Infy-data-schema-fix
HarishGangula Oct 24, 2024
3827e9f
Fix code scanning alert no. 98: Loop bound injection
HarishGangula Oct 24, 2024
856e6d7
#I285: lint issues fixed
HarishGangula Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
const request = <DatasetSchemeRequest>req.body.request
const dataSchemaSpec = schemaGenerate(request.data, request.config)
ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec });

}

const schemaGenerate = (sample: Map<string, any>[], config: Record<string, any>): any => {
Expand All @@ -46,13 +46,14 @@
result.schema = removeFormats(result.schema)
return result
} else {
let schema = isBatch ? schemaInference.inferBatchSchema(<Map<string, any>[]>sample, extractionKey) : schemaInference.inferSchema(sample);
let { schema, removedKeys } = isBatch ? schemaInference.inferBatchSchema(<Map<string, any>[]>sample, extractionKey) : schemaInference.inferSchema(sample);

Check failure on line 49 in api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts

View workflow job for this annotation

GitHub Actions / test-cases 

'removedKeys' is never reassigned. Use 'const' instead
schema = schemaArrayValidator.validate(schema)
const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema)
rollupInfo = schemaCardinalityAnalyser.analyse()
const result = process(schema, dataset)
result.schema = removeNonIndexColumns(result.schema)
result.schema = removeFormats(result.schema)
!_.isEmpty(removedKeys) && _.set(result, "removedKeys", removedKeys)
return result
}
}
Expand Down
54 changes: 39 additions & 15 deletions api-service/src/services/SchemaGenerateService/DataSchemaService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException";

const DATE_FORMATS = [
"MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD",
"MM/DD/YYYY", "DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD",
"DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss",
"YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss",
"DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS",
"YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS",
"DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ",
"YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss",
"DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS",
"YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS",
"DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ",
"YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ",
"YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS",
"MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ",
"YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ",
"MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss",
"YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ",
"MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss",
"YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ",
"YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ",
"MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ",
Expand All @@ -38,23 +38,47 @@
export class SchemaInference {

public inferSchema(sample: any) {
const schema = _.map(sample, (value): any => this.validateEpoch(inferSchema(value).toJSONSchema({ includeSchema: true }), value, "properties"))
return schema
const removedAllKeys: any[] = []
const schema = _.map(sample, (value): any => {
Fixed Show fixed Hide fixed
const { cleanedData, removedKeys } = this.removeEmpty(value)
removedAllKeys.push(...removedKeys)
return this.validateEpoch(inferSchema(cleanedData).toJSONSchema({ includeSchema: true }), cleanedData, "properties")
})
return { schema, removedKeys: removedAllKeys }
}

public inferBatchSchema(sample: Map<string, any>[], extractionKey: string) {
return _.flatMap(sample, (value) => {
const removedAllKeys: any[] = []
const schema = _.flatMap(sample, (value) => {
if (extractionKey) {
const extracted = _.get(value, extractionKey);
if (extracted) {
return this.inferSchema(extracted);
const { schema, removedKeys } = this.inferSchema(extracted);
removedAllKeys.push(...removedKeys)
return schema
} else {
throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST);
}
} else {
throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST);
}
})
return { schema, removedKeys: removedAllKeys }
}
// Only removes empty object and array at all the levels
private removeEmpty(data: any, parentKey = '', removedKeys: any[] = []) {

Check failure on line 69 in api-service/src/services/SchemaGenerateService/DataSchemaService.ts

View workflow job for this annotation

GitHub Actions / test-cases 

Strings must use doublequote
Object.keys(data).forEach((key) => {
const value = data[key];
const fullKey = parentKey ? `${parentKey}.${key}` : key;
if (typeof value === 'object' && value !== null) {

Check failure on line 73 in api-service/src/services/SchemaGenerateService/DataSchemaService.ts

View workflow job for this annotation

GitHub Actions / test-cases 

Strings must use doublequote
this.removeEmpty(value, fullKey, removedKeys);
if (_.isEmpty(value)) {
delete data[key];
removedKeys.push({ "key": fullKey, value });
}
}
});
return { cleanedData: data, removedKeys };
}

private validateEpoch(schema: any, sample: any, path: any) {
Expand All @@ -67,7 +91,7 @@
if (isValidTimestamp) {
_.set(schema, `${path}.${key}.format`, type)
}
else if(format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) {
else if (format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) {
_.unset(schema, `${path}.${key}.format`)
}
});
Expand All @@ -79,7 +103,7 @@
const epochRegex = /^\d+$/ig;
switch (dataType) {
case "string":
if(epochRegex.test(value)){
if (epochRegex.test(value)) {
const parsedValue = parseInt(value, 10);
// Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds
return {
Expand All @@ -93,12 +117,12 @@
case "number":
// Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds
return {
isValidTimestamp: value >= 946684800 && moment(value).isValid(),
isValidTimestamp: value >= 946684800 && moment(value).isValid(),
type: "epoch"
};
default:
return {
isValidTimestamp: false,
isValidTimestamp: false,
type: ""
};
}
Expand Down
Loading