Skip to content

Commit

Permalink
chore(cfnspec): improve error meesaging when spec validation fails (#…
Browse files Browse the repository at this point in the history 
…16991)

Print the node of the CFN spec where validation fails, so the erroneous
part of the spec can be located.

Currently, when a Property in a PropertyType has two "Type" fields
specified, the message looks like below. Only the resource type being
validated is printed.

```
FAIL test/filtered-specification.test.js (8.522 s)
  filteredSpecification("AWS::Synthetics::Canary")

  expect(received).toEqual(expected)

  - Expected  - 1
  + Received  + 3

  - Array []
  + Array [
  +   "Type",
  + ]
```

With this change, the message will look like -

```
FAIL test/filtered-specification.test.js (8.522 s)
  filteredSpecification("AWS::Synthetics::Canary") › PropertyType AWS::Synthetics::Canary.ArtifactConfig › Property S3Encryption

  expect(received).toEqual(expected)

  - Expected  - 1
  + Received  + 3

  - Array []
  + Array [
  +   "Type",
  + ]
```


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
Niranjan Jayakar authored Oct 15, 2021
1 parent 55fbc86 commit fbb49fe
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 105 deletions.
2 changes: 1 addition & 1 deletion packages/@aws-cdk/cfnspec/test/filtered-specification.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ test('filteredSpecification(s => s.startsWith("AWS::S3::")', () => {
});

for (const name of resourceTypes().sort()) {
test(`filteredSpecification(${JSON.stringify(name)})`, () => {
describe(`filteredSpecification(${JSON.stringify(name)})`, () => {
const filteredSpec = filteredSpecification(name);
expect(filteredSpec).not.toEqual(specification);
expect(filteredSpec.ResourceTypes).not.toEqual({});
Expand Down
217 changes: 113 additions & 104 deletions packages/@aws-cdk/cfnspec/test/spec-validators.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable jest/no-export */
import * as schema from '../lib/schema';

export function validateSpecification(specification: schema.Specification) {
Expand All @@ -7,26 +8,30 @@ export function validateSpecification(specification: schema.Specification) {

function validateResourceTypes(specification: schema.Specification) {
for (const typeName of Object.keys(specification.ResourceTypes)) {
expect(typeName).toBeTruthy();
const type = specification.ResourceTypes[typeName];
expect(type.Documentation).not.toBeNull();
if (type.ScrutinyType) {
expect(schema.isResourceScrutinyType(type.ScrutinyType)).toBeTruthy();
}
if (type.Properties) { validateProperties(typeName, type.Properties, specification); }
if (type.Attributes) { validateAttributes(typeName, type.Attributes, specification); }
describe(typeName, () => {
expect(typeName).toBeTruthy();
const type = specification.ResourceTypes[typeName];
expect(type.Documentation).not.toBeNull();
if (type.ScrutinyType) {
expect(schema.isResourceScrutinyType(type.ScrutinyType)).toBeTruthy();
}
if (type.Properties) { validateProperties(typeName, type.Properties, specification); }
if (type.Attributes) { validateAttributes(typeName, type.Attributes, specification); }
});
}
}

function validatePropertyTypes(specification: schema.Specification) {
for (const typeName of Object.keys(specification.PropertyTypes)) {
expect(typeName).toBeTruthy();
const type = specification.PropertyTypes[typeName];
if (schema.isRecordType(type)) {
validateProperties(typeName, type.Properties, specification);
} else {
validateProperties(typeName, { '<this>': type }, specification);
}
describe(`PropertyType ${typeName}`, () => {
expect(typeName).toBeTruthy();
const type = specification.PropertyTypes[typeName];
if (schema.isRecordType(type)) {
validateProperties(typeName, type.Properties, specification);
} else {
validateProperties(typeName, { '<this>': type }, specification);
}
});
}
}

Expand All @@ -37,85 +42,87 @@ function validateProperties(
const expectedKeys = ['Documentation', 'Required', 'UpdateType', 'ScrutinyType'];
for (const name of Object.keys(properties)) {

const property = properties[name];
expect(property.Documentation).not.toEqual('');
expect(!property.UpdateType || schema.isUpdateType(property.UpdateType)).toBeTruthy();
if (property.ScrutinyType !== undefined) {
expect(schema.isPropertyScrutinyType(property.ScrutinyType)).toBeTruthy();
}
test(`Property ${name}`, () => {
const property = properties[name];
expect(property.Documentation).not.toEqual('');
expect(!property.UpdateType || schema.isUpdateType(property.UpdateType)).toBeTruthy();
if (property.ScrutinyType !== undefined) {
expect(schema.isPropertyScrutinyType(property.ScrutinyType)).toBeTruthy();
}

if (schema.isPrimitiveProperty(property)) {
expect(schema.isPrimitiveType(property.PrimitiveType)).toBeTruthy();
expectedKeys.push('PrimitiveType');

if (schema.isPrimitiveProperty(property)) {
expect(schema.isPrimitiveType(property.PrimitiveType)).toBeTruthy();
expectedKeys.push('PrimitiveType');
} else if (schema.isPrimitiveListProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'PrimitiveItemType');
expect(schema.isPrimitiveType(property.PrimitiveItemType)).toBeTruthy();

} else if (schema.isPrimitiveListProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'PrimitiveItemType');
expect(schema.isPrimitiveType(property.PrimitiveItemType)).toBeTruthy();
} else if (schema.isPrimitiveMapProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'PrimitiveItemType', 'Type');
expect(schema.isPrimitiveType(property.PrimitiveItemType)).toBeTruthy();
expect(property.DuplicatesAllowed).toBeFalsy();

} else if (schema.isPrimitiveMapProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'PrimitiveItemType', 'Type');
expect(schema.isPrimitiveType(property.PrimitiveItemType)).toBeTruthy();
expect(property.DuplicatesAllowed).toBeFalsy();
} else if (schema.isComplexListProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'Type');
expect(property.ItemType).toBeTruthy();
if (property.ItemType !== 'Tag') {
const fqn = `${typeName.split('.')[0]}.${property.ItemType}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
}

} else if (schema.isComplexListProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'Type');
expect(property.ItemType).toBeTruthy();
if (property.ItemType !== 'Tag') {
} else if (schema.isMapOfStructsProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'Type');
expect(property.ItemType).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${property.ItemType}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
}
expect(property.DuplicatesAllowed).toBeFalsy();

} else if (schema.isMapOfListsOfPrimitivesProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'PrimitiveItemItemType', 'Type');
expect(schema.isPrimitiveType(property.PrimitiveItemItemType)).toBeTruthy();
expect(property.DuplicatesAllowed).toBeFalsy();

} else if (schema.isComplexProperty(property)) {
expectedKeys.push('Type');
expect(property.Type).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${property.Type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();

} else if (schema.isMapOfStructsProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'Type');
expect(property.ItemType).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${property.ItemType}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
expect(property.DuplicatesAllowed).toBeFalsy();

} else if (schema.isMapOfListsOfPrimitivesProperty(property)) {
expectedKeys.push('Type', 'DuplicatesAllowed', 'ItemType', 'PrimitiveItemItemType', 'Type');
expect(schema.isPrimitiveType(property.PrimitiveItemItemType)).toBeTruthy();
expect(property.DuplicatesAllowed).toBeFalsy();

} else if (schema.isComplexProperty(property)) {
expectedKeys.push('Type');
expect(property.Type).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${property.Type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();

} else if (schema.isUnionProperty(property)) {
expectedKeys.push('PrimitiveTypes', 'PrimitiveItemTypes', 'ItemTypes', 'Types');
if (property.PrimitiveTypes) {
for (const type of property.PrimitiveTypes) {
expect(schema.isPrimitiveType(type)).toBeTruthy();
} else if (schema.isUnionProperty(property)) {
expectedKeys.push('PrimitiveTypes', 'PrimitiveItemTypes', 'ItemTypes', 'Types');
if (property.PrimitiveTypes) {
for (const type of property.PrimitiveTypes) {
expect(schema.isPrimitiveType(type)).toBeTruthy();
}
}
}
if (property.ItemTypes) {
for (const type of property.ItemTypes) {
const fqn = `${typeName.split('.')[0]}.${type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
if (property.ItemTypes) {
for (const type of property.ItemTypes) {
const fqn = `${typeName.split('.')[0]}.${type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
}
}
}
if (property.Types) {
for (const type of property.Types) {
const fqn = `${typeName.split('.')[0]}.${type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
if (property.Types) {
for (const type of property.Types) {
const fqn = `${typeName.split('.')[0]}.${type}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
}
}
}

} else {
// eslint-disable-next-line no-console
console.error(`${typeName}.Properties.${name} does not declare a type.` +
`Property definition is: ${JSON.stringify(property, undefined, 2)}`);
expect(false).toBeTruthy();
}
} else {
// eslint-disable-next-line no-console
console.error(`${typeName}.Properties.${name} does not declare a type.` +
`Property definition is: ${JSON.stringify(property, undefined, 2)}`);
expect(false).toBeTruthy();
}

expect(without(Object.keys(property), expectedKeys)).toEqual([]);
expect(without(Object.keys(property), expectedKeys)).toEqual([]);
});
}
}

Expand All @@ -124,29 +131,31 @@ function validateAttributes(
attributes: { [name: string]: schema.Attribute },
specification: schema.Specification) {
for (const name of Object.keys(attributes)) {
const attribute = attributes[name];
expect(('Type' in attribute)).not.toEqual(('PrimitiveType' in attribute));
if (schema.isPrimitiveAttribute(attribute)) {
expect(schema.isListAttribute(attribute)).toBeFalsy();
expect(schema.isPrimitiveType(attribute.PrimitiveType)).toBeTruthy();
expect(('PrimitiveItemType' in attribute)).toBeFalsy();
expect(('ItemType' in attribute)).toBeFalsy();
} else if (schema.isPrimitiveListAttribute(attribute)) {
expect(schema.isComplexListAttribute(attribute)).toBeFalsy();
expect(schema.isPrimitiveType(attribute.PrimitiveItemType)).toBeTruthy();
expect(('ItemType' in attribute)).toBeFalsy();
} else if (schema.isComplexListAttribute(attribute)) {
expect(attribute.ItemType).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${attribute.ItemType}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
expect(('PrimitiveItemType' in attribute)).toBeFalsy();
} else if (schema.isPrimitiveMapAttribute(attribute)) {
expect(schema.isPrimitiveType(attribute.PrimitiveItemType)).toBeTruthy();
expect(('ItemType' in attribute)).toBeFalsy();
} else {
expect(false).toBeTruthy(); // `${typeName}.Attributes.${name} has a valid type`);
}
test(`Attribute ${name}`, () => {
const attribute = attributes[name];
expect(('Type' in attribute)).not.toEqual(('PrimitiveType' in attribute));
if (schema.isPrimitiveAttribute(attribute)) {
expect(schema.isListAttribute(attribute)).toBeFalsy();
expect(schema.isPrimitiveType(attribute.PrimitiveType)).toBeTruthy();
expect(('PrimitiveItemType' in attribute)).toBeFalsy();
expect(('ItemType' in attribute)).toBeFalsy();
} else if (schema.isPrimitiveListAttribute(attribute)) {
expect(schema.isComplexListAttribute(attribute)).toBeFalsy();
expect(schema.isPrimitiveType(attribute.PrimitiveItemType)).toBeTruthy();
expect(('ItemType' in attribute)).toBeFalsy();
} else if (schema.isComplexListAttribute(attribute)) {
expect(attribute.ItemType).toBeTruthy();
const fqn = `${typeName.split('.')[0]}.${attribute.ItemType}`;
const resolvedType = specification.PropertyTypes && specification.PropertyTypes[fqn];
expect(resolvedType).toBeTruthy();
expect(('PrimitiveItemType' in attribute)).toBeFalsy();
} else if (schema.isPrimitiveMapAttribute(attribute)) {
expect(schema.isPrimitiveType(attribute.PrimitiveItemType)).toBeTruthy();
expect(('ItemType' in attribute)).toBeFalsy();
} else {
expect(false).toBeTruthy(); // `${typeName}.Attributes.${name} has a valid type`);
}
});
}
}

Expand Down

0 comments on commit fbb49fe

Please sign in to comment.