Skip to content

Commit

Permalink
Add selector compliance tests
Browse files Browse the repository at this point in the history
  • Loading branch information
AndrewFossAWS committed Mar 3, 2023
1 parent 5e2b310 commit 2dc880a
Show file tree
Hide file tree
Showing 15 changed files with 1,008 additions and 0 deletions.
70 changes: 70 additions & 0 deletions docs/source-1.0/spec/core/selectors.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1648,3 +1648,73 @@ Selectors are defined by the following ABNF_ grammar.
.. _ABNF: https://tools.ietf.org/html/rfc5234
.. _set: https://en.wikipedia.org/wiki/Set_(abstract_data_type)
Compliance Tests
================
Selector compliance tests are used to verify the behavior of selectors. Each compliance test is written as a Smithy file
and includes a :ref:`metadata <metadata>` called ``selectorTests``. This metadata contains a list of test cases, each including a selector,
the expected matched shapes, and additional configuration options. The test case contains the following properties:
.. list-table::
:header-rows: 1
:widths: 10 20 70
* - Property
- Type
- Description
* - selector
- ``string``
- **REQUIRED** The selector to match shapes within the smithy model
* - matches
- ``list<shape ID>``
- **REQUIRED** The expected shapes ID of the matched shapes
* - skipPreludeShapes
- ``boolean``
- Skip :ref:`prelude shapes <prelude>` when comparing the expected shapes and the actual shapes returned from the selector
Below is an example selector compliance test:
.. code-block:: none
$version: "1.0"
metadata selectorTests = [
{
selector: "[trait|length|min > 1]"
matches: [
smithy.example#AtLeastTen
]
}
{
selector: "[trait|length|min >= 1]"
skipPreludeShapes: true
matches: [
smithy.example#AtLeastOne
smithy.example#AtLeastTen
]
}
{
selector: "[trait|length|min < 2]"
skipPreludeShapes: true
matches: [
smithy.example#AtLeastOne
]
}
]
namespace smithy.example
@length(min: 1)
string AtLeastOne
@length(max: 5)
string AtMostFive
@length(min: 10)
string AtLeastTen
The compliance tests can also be accessed in the
https://github.com/awslabs/smithy/tree/main/smithy-model/src/test/resources/software/amazon/smithy/model/selector/cases
directory of the Smithy Github repository.
70 changes: 70 additions & 0 deletions docs/source-2.0/spec/selectors.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1664,3 +1664,73 @@ Selectors are defined by the following :rfc:`ABNF <5234>` grammar.
SelectorVariableGet :"${" `smithy:Identifier` "}"
.. _set: https://en.wikipedia.org/wiki/Set_(abstract_data_type)
Compliance Tests
================
Selector compliance tests are used to verify the behavior of selectors. Each compliance test is written as a Smithy file
and includes a :ref:`metadata <metadata>` called ``selectorTests``. This metadata contains a list of test cases, each including a selector,
the expected matched shapes, and additional configuration options. The test case contains the following properties:
.. list-table::
:header-rows: 1
:widths: 10 20 70
* - Property
- Type
- Description
* - selector
- ``string``
- **REQUIRED** The selector to match shapes within the smithy model
* - matches
- ``list<shape ID>``
- **REQUIRED** The expected shapes ID of the matched shapes
* - skipPreludeShapes
- ``boolean``
- Skip :ref:`prelude shapes <prelude>` when comparing the expected shapes and the actual shapes returned from the selector
Below is an example selector compliance test:
.. code-block:: none
$version: "2.0"
metadata selectorTests = [
{
selector: "[trait|length|min > 1]"
matches: [
smithy.example#AtLeastTen
]
}
{
selector: "[trait|length|min >= 1]"
skipPreludeShapes: true
matches: [
smithy.example#AtLeastOne
smithy.example#AtLeastTen
]
}
{
selector: "[trait|length|min < 2]"
skipPreludeShapes: true
matches: [
smithy.example#AtLeastOne
]
}
]
namespace smithy.example
@length(min: 1)
string AtLeastOne
@length(max: 5)
string AtMostFive
@length(min: 10)
string AtLeastTen
The compliance tests can also be accessed in the
https://github.com/awslabs/smithy/tree/main/smithy-model/src/test/resources/software/amazon/smithy/model/selector/cases
directory of the Smithy Github repository.
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package software.amazon.smithy.model.selector;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.loader.Prelude;
import software.amazon.smithy.model.node.ObjectNode;
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.ShapeId;

public final class SelectorRunnerTest {

@ParameterizedTest(name = "{0}")
@MethodSource("source")
public void selectorTest(Path filename) {
Model model = Model.assembler().addImport(filename).assemble().unwrap();
List<ObjectNode> tests = findTestCases(model);

for (ObjectNode test : tests) {
Selector selector = Selector.parse(test.expectStringMember("selector").getValue());
boolean skipPrelude = test.getBooleanMemberOrDefault("skipPreludeShapes", false);

Set<ShapeId> expectedMatches = new HashSet<>(new HashSet<>(test.expectArrayMember("matches")
.getElementsAs(n -> n.expectStringNode("Each element of matches must be an ID").expectShapeId())))
.stream()
.filter(shapeId -> {
String namespace = shapeId.getNamespace();
return !namespace.contains(Prelude.NAMESPACE)
|| (!skipPrelude && namespace.contains(Prelude.NAMESPACE));
})
.collect(Collectors.toSet());

Set<ShapeId> actualMatches = selector.shapes(model)
.map(Shape::getId)
.filter(shapeId -> {
String namespace = shapeId.getNamespace();
return !namespace.contains(Prelude.NAMESPACE)
|| (!skipPrelude && namespace.contains(Prelude.NAMESPACE));
})
.collect(Collectors.toSet());

if (!expectedMatches.equals(actualMatches)) {
failTest(filename, test, expectedMatches, actualMatches);
}
}
}

private List<ObjectNode> findTestCases(Model model) {
return model.getMetadataProperty("selectorTests")
.orElseThrow(() -> new IllegalArgumentException("Missing selectorTests metadata key"))
.expectArrayNode("selectorTests must be an array")
.getElementsAs(ObjectNode.class);
}

private void failTest(Path filename, ObjectNode test, Set<ShapeId> expectedMatches, Set<ShapeId> actualMatches) {
String selector = test.expectStringMember("selector").getValue();
Set<ShapeId> missing = new TreeSet<>(expectedMatches);
missing.removeAll(actualMatches);

Set<ShapeId> extra = new TreeSet<>(actualMatches);
extra.removeAll(expectedMatches);

StringBuilder error = new StringBuilder("Selector ")
.append(selector)
.append(" test case failed.\n");

if (!missing.isEmpty()) {
error.append("The following shapes were not matched: ").append(missing).append(".\n");
}

if (!extra.isEmpty()) {
error.append("The following shapes were matched unexpectedly: ").append(extra).append(".\n");
}

test.getStringMember("documentation")
.ifPresent(docs -> error.append('(').append(docs.getValue()).append(")"));

Assertions.fail(error.toString());
}

public static List<Path> source() throws Exception {
List<Path> paths = new ArrayList<>();
try (Stream<Path> files = Files.walk(Paths.get(SelectorRunnerTest.class.getResource("cases").toURI()))) {
files
.filter(Files::isRegularFile)
.filter(file -> {
String filename = file.toString();
return filename.endsWith(".smithy") || filename.endsWith(".json");
})
.forEach(paths::add);
} catch (IOException e) {
throw new RuntimeException("Error loading models for selector runner", e);
}

return paths;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
$version: "2.0"

metadata selectorTests = [
{
selector: "[trait|enum]"
matches: [
smithy.example#SimpleEnum
smithy.example#EnumWithTags
]
},
{
selector: "[trait|enum|(values)|tags|(values)]"
matches: [
smithy.example#EnumWithTags
]
}
]

namespace smithy.example

@deprecated
string NoMatch

@enum([
{name: "foo", value: "foo"}
{name: "baz", value: "baz"}
])
string SimpleEnum

@enum([
{name: "foo", value: "foo", tags: ["a"]}
{name: "baz", value: "baz"}
{name: "spam", value: "spam", tags: []}
])
string EnumWithTags
Loading

0 comments on commit 2dc880a

Please sign in to comment.