Skip to content
This repository has been archived by the owner on Feb 22, 2020. It is now read-only.

Commit

Permalink
Merge branch 'develop' into feature/101_shacl_w3c_tests
Browse files Browse the repository at this point in the history
* develop: (25 commits)
  eclipse-rdf4j/rdf4j#1319 added a test
  eclipse-rdf4j/rdf4j#1319 super.prepare() as final step to stop it from locking until commit()
  fix init and fix test
  eclipse-rdf4j/rdf4j#1315 make parallel validation disabled by default and log warning
  typo Jeen found
  restore javadocs
  fixup pom
  initialize() still needs to call init() even though initialize() is deprecated
  eclipse-rdf4j/rdf4j#1314 support for NativeStore
  eclipse-rdf4j/rdf4j#1312 more caching
  eclipse-rdf4j/rdf4j#1312 fix copyright headers and cleanup reuse code in nodeshape
  eclipse-rdf4j/rdf4j#1306 support for sh:class and sh:or
  eclipse-rdf4j/rdf4j#1306 support in maxCount
  eclipse-rdf4j/rdf4j#1306 support minCount
  eclipse-rdf4j/rdf4j#1306 initial implementation
  optimized imports
  eclipse-rdf4j/rdf4j#1307 some cleanup
  eclipse-rdf4j/rdf4j#1307 fix erronous close
  eclipse-rdf4j/rdf4j#1307 fix close issues
  eclipse-rdf4j/rdf4j#1307 fixed illegalstateexception
  ...
  • Loading branch information
hmottestad committed Mar 7, 2019
2 parents 46b646f + f10fe12 commit 7c81edb
Show file tree
Hide file tree
Showing 145 changed files with 2,605 additions and 743 deletions.
6 changes: 6 additions & 0 deletions shacl/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rdf4j-sail-nativerdf</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

</dependencies>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,19 @@
import org.eclipse.rdf4j.sail.SailConnection;
import org.eclipse.rdf4j.sail.shacl.ShaclSailConnection;
import org.eclipse.rdf4j.sail.shacl.SourceConstraintComponent;
import org.eclipse.rdf4j.sail.shacl.planNodes.BufferedSplitter;
import org.eclipse.rdf4j.sail.shacl.planNodes.BufferedTupleFromFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.BufferedPlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.BulkedExternalInnerJoin;
import org.eclipse.rdf4j.sail.shacl.planNodes.BulkedExternalLeftOuterJoin;
import org.eclipse.rdf4j.sail.shacl.planNodes.DirectTupleFromFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.EnrichWithShape;
import org.eclipse.rdf4j.sail.shacl.planNodes.ExternalTypeFilterNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.InnerJoin;
import org.eclipse.rdf4j.sail.shacl.planNodes.LoggingNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.ModifyTuple;
import org.eclipse.rdf4j.sail.shacl.planNodes.PlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.PushBasedLoggingNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.Select;
import org.eclipse.rdf4j.sail.shacl.planNodes.Sort;
import org.eclipse.rdf4j.sail.shacl.planNodes.TupleLengthFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnBufferedPlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnionNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.Unique;
import org.slf4j.Logger;
Expand Down Expand Up @@ -79,41 +77,33 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
return new EnrichWithShape(invalidTuplesDueToDataAddedThatMatchesTargetOrPath, this);
} else {

PlanNode addedByShape1 = new LoggingNode(nodeShape.getPlanAddedStatements(shaclSailConnection, nodeShape), "");


BufferedSplitter bufferedAddedByShape = new BufferedSplitter(addedByShape1);

PlanNode addedByPath = new LoggingNode(getPlanAddedStatements(shaclSailConnection, nodeShape), "");


BufferedTupleFromFilter discardedRight = new BufferedTupleFromFilter();

// join all added by type and path
PlanNode leftOuterJoin = new LoggingNode(new InnerJoin(bufferedAddedByShape.getPlanNode(), addedByPath, null,new PushBasedLoggingNode(discardedRight)), "");
InnerJoin innerJoinHolder = new InnerJoin(new LoggingNode(nodeShape.getPlanAddedStatements(shaclSailConnection, nodeShape), ""), addedByPath);
PlanNode innerJoin = new LoggingNode(innerJoinHolder.getJoined(BufferedPlanNode.class), "");
PlanNode discardedRight = new LoggingNode(innerJoinHolder.getDiscardedRight(BufferedPlanNode.class), "");

if (nodeShape instanceof TargetClass) {
PlanNode typeFilterPlan = new LoggingNode(((TargetClass) nodeShape).getTypeFilterPlan(shaclSailConnection, discardedRight), "");
PlanNode typeFilterPlan = new LoggingNode(nodeShape.getTargetFilter(shaclSailConnection, discardedRight), "");

leftOuterJoin = new LoggingNode(new Unique(new UnionNode(leftOuterJoin, typeFilterPlan)), "");
innerJoin = new LoggingNode(new Unique(new UnionNode(innerJoin, typeFilterPlan)), "");

}

// also add anything that matches the path from the previousConnection, eg. if you add ":peter a foaf:Person", and ":peter foaf:knows :steve" is already added
PlanNode bulkedExternalLeftOuter = new LoggingNode(new BulkedExternalLeftOuterJoin(bufferedAddedByShape.getPlanNode(), shaclSailConnection, path.getQuery("?a", "?c", null), true), "");
PlanNode bulkedExternalLeftOuter = new LoggingNode(new BulkedExternalLeftOuterJoin(new LoggingNode(nodeShape.getPlanAddedStatements(shaclSailConnection, nodeShape), ""), shaclSailConnection, path.getQuery("?a", "?c", null), true), "");

// only get tuples that came from the first or the leftOuterJoin or bulkedExternalLeftOuter,
// only get tuples that came from the first or the innerJoin or bulkedExternalLeftOuter,
// we don't care if you added ":peter a foaf:Person" and nothing else and there is nothing else in the underlying sail
DirectTupleFromFilter joined = new DirectTupleFromFilter();
new TupleLengthFilter(new UnionNode(leftOuterJoin, bulkedExternalLeftOuter), joined, null, 2, false);
PlanNode joined = new TupleLengthFilter(new UnionNode(innerJoin, bulkedExternalLeftOuter), 2, false).getTrueNode(UnBufferedPlanNode.class);

// filter by type against addedStatements, this is an optimization for when you add the type statement in the same transaction
PlanNode addedStatementsTypeFilter = new LoggingNode(new ExternalTypeFilterNode(addedStatements, classResource, joined, 1, false), "");

// filter by type against the base sail
PlanNode invalidTuplesDueToDataAddedThatMatchesTargetOrPath = new LoggingNode(new ExternalTypeFilterNode(shaclSailConnection, classResource, addedStatementsTypeFilter, 1, false), "");

if(shaclSailConnection.stats.hasRemoved()) {
if (shaclSailConnection.stats.hasRemoved()) {

// Handle when a type statement has been removed, first get all removed type statements that match the classResource for this shape
PlanNode removedTypeStatements = new LoggingNode(new Select(shaclSailConnection.getRemovedStatements(), "?a a <" + classResource + ">"), "removedTypeStatements");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new DatatypeFilter(parent, trueNode, falseNode, datatype),
(parent) -> new DatatypeFilter(parent, datatype),
this,
overrideTargetNode
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new LanguageInFilter(parent, trueNode, falseNode, languageIn),
(parent) -> new LanguageInFilter(parent, languageIn),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
import org.eclipse.rdf4j.sail.shacl.ShaclSailConnection;
import org.eclipse.rdf4j.sail.shacl.SourceConstraintComponent;
import org.eclipse.rdf4j.sail.shacl.planNodes.BufferedTupleFromFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.BufferedPlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.BulkedExternalLeftOuterJoin;
import org.eclipse.rdf4j.sail.shacl.planNodes.DirectTupleFromFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.EnrichWithShape;
import org.eclipse.rdf4j.sail.shacl.planNodes.GroupByCount;
import org.eclipse.rdf4j.sail.shacl.planNodes.LoggingNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.MaxCountFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.PlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.TrimTuple;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnBufferedPlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnionNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.Unique;
import org.slf4j.Logger;
Expand Down Expand Up @@ -53,15 +53,13 @@ public String toString() {
@Override
public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeShape, boolean printPlans, PlanNode overrideTargetNode) {

if(overrideTargetNode != null){
if (overrideTargetNode != null) {
PlanNode bulkedExternalLeftOuterJoin = new LoggingNode(new BulkedExternalLeftOuterJoin(overrideTargetNode, shaclSailConnection, path.getQuery("?a", "?c", null), false), "");
PlanNode groupByCount = new LoggingNode(new GroupByCount(bulkedExternalLeftOuterJoin), "");

DirectTupleFromFilter directTupleFromFilter = new DirectTupleFromFilter();
PlanNode directTupleFromFilter = new LoggingNode(new MaxCountFilter(groupByCount, maxCount).getFalseNode(UnBufferedPlanNode.class), "");

new MaxCountFilter(groupByCount, null, directTupleFromFilter, maxCount);

if(printPlans){
if (printPlans) {
String planAsGraphvizDot = getPlanAsGraphvizDot(directTupleFromFilter, shaclSailConnection);
logger.info(planAsGraphvizDot);
}
Expand All @@ -74,19 +72,17 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS

PlanNode planAddedStatements1 = new LoggingNode(super.getPlanAddedStatements(shaclSailConnection, nodeShape), "");


if (nodeShape instanceof TargetClass) {
planAddedStatements1 = new LoggingNode(((TargetClass) nodeShape).getTypeFilterPlan(shaclSailConnection, planAddedStatements1), "");
}
planAddedStatements1 = new LoggingNode(nodeShape.getTargetFilter(shaclSailConnection, planAddedStatements1), "");

PlanNode mergeNode = new LoggingNode(new UnionNode(planAddedStatements, planAddedStatements1), "");

PlanNode groupByCount1 = new LoggingNode(new GroupByCount(mergeNode), "");

BufferedTupleFromFilter validValues = new BufferedTupleFromFilter();
BufferedTupleFromFilter invalidValues = new BufferedTupleFromFilter();

new MaxCountFilter(groupByCount1, validValues, invalidValues, maxCount);
MaxCountFilter maxCountFilter = new MaxCountFilter(groupByCount1, maxCount);

PlanNode validValues = maxCountFilter.getTrueNode(BufferedPlanNode.class);
PlanNode invalidValues = maxCountFilter.getFalseNode(BufferedPlanNode.class);

PlanNode trimmed = new LoggingNode(new TrimTuple(validValues, 0, 1), "");

Expand All @@ -96,13 +92,11 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS

PlanNode groupByCount = new LoggingNode(new GroupByCount(bulkedExternalLeftOuterJoin), "");

DirectTupleFromFilter directTupleFromFilter = new DirectTupleFromFilter();

new MaxCountFilter(groupByCount, null, directTupleFromFilter, maxCount);
PlanNode directTupleFromFilter = new MaxCountFilter(groupByCount, maxCount).getFalseNode(UnBufferedPlanNode.class);

PlanNode mergeNode1 = new UnionNode(new LoggingNode(directTupleFromFilter, ""), new LoggingNode(invalidValues, ""));

if(printPlans){
if (printPlans) {
String planAsGraphvizDot = getPlanAsGraphvizDot(mergeNode1, shaclSailConnection);
logger.info(planAsGraphvizDot);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new LiteralComparatorFilter(parent, trueNode, falseNode, maxExclusive, value -> value > 0),
(parent) -> new LiteralComparatorFilter(parent, maxExclusive, value -> value > 0),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new LiteralComparatorFilter(parent, trueNode, falseNode, maxInclusive, value -> value >= 0),
(parent) -> new LiteralComparatorFilter(parent, maxInclusive, value -> value >= 0),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new MaxLengthFilter(parent, trueNode, falseNode, maxLength),
(parent) -> new MaxLengthFilter(parent, maxLength),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@
import org.eclipse.rdf4j.sail.shacl.ShaclSailConnection;
import org.eclipse.rdf4j.sail.shacl.SourceConstraintComponent;
import org.eclipse.rdf4j.sail.shacl.planNodes.BulkedExternalLeftOuterJoin;
import org.eclipse.rdf4j.sail.shacl.planNodes.DirectTupleFromFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.EnrichWithShape;
import org.eclipse.rdf4j.sail.shacl.planNodes.GroupByCount;
import org.eclipse.rdf4j.sail.shacl.planNodes.LoggingNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.MinCountFilter;
import org.eclipse.rdf4j.sail.shacl.planNodes.PlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.Select;
import org.eclipse.rdf4j.sail.shacl.planNodes.TrimTuple;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnBufferedPlanNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.UnionNode;
import org.eclipse.rdf4j.sail.shacl.planNodes.Unique;
import org.slf4j.Logger;
Expand Down Expand Up @@ -56,12 +55,11 @@ public String toString() {
@Override
public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeShape, boolean printPlans, PlanNode overrideTargetNode) {

if(overrideTargetNode != null){
if (overrideTargetNode != null) {
PlanNode allStatements = new LoggingNode(new BulkedExternalLeftOuterJoin(overrideTargetNode, shaclSailConnection, path.getQuery("?a", "?c", null), false), "");
PlanNode groupBy = new LoggingNode(new GroupByCount(allStatements), "");

DirectTupleFromFilter filteredStatements = new DirectTupleFromFilter();
new MinCountFilter(groupBy, null, filteredStatements, minCount);
PlanNode filteredStatements = new MinCountFilter(groupBy, minCount).getFalseNode(UnBufferedPlanNode.class);

if (printPlans) {
String planAsGraphvizDot = getPlanAsGraphvizDot(filteredStatements, shaclSailConnection);
Expand All @@ -77,11 +75,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
if (!optimizeWhenNoStatementsRemoved || shaclSailConnection.stats.hasRemoved()) {
PlanNode planRemovedStatements = new LoggingNode(new TrimTuple(new LoggingNode(super.getPlanRemovedStatements(shaclSailConnection, nodeShape), ""), 0, 1), "");

PlanNode filteredPlanRemovedStatements = planRemovedStatements;

if (nodeShape instanceof TargetClass) {
filteredPlanRemovedStatements = new LoggingNode(((TargetClass) nodeShape).getTypeFilterPlan(shaclSailConnection, planRemovedStatements), "");
}
PlanNode filteredPlanRemovedStatements = new LoggingNode(nodeShape.getTargetFilter(shaclSailConnection, planRemovedStatements), "");

PlanNode planAddedStatements = new LoggingNode(nodeShape.getPlanAddedStatements(shaclSailConnection, nodeShape), "");

Expand All @@ -92,9 +86,8 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS

PlanNode planAddedStatements1 = super.getPlanAddedStatements(shaclSailConnection, nodeShape);

if (nodeShape instanceof TargetClass) {
planAddedStatements1 = new LoggingNode(((TargetClass) nodeShape).getTypeFilterPlan(shaclSailConnection, planAddedStatements1), "");
}
planAddedStatements1 = new LoggingNode((nodeShape).getTargetFilter(shaclSailConnection, planAddedStatements1), "");

topNode = new LoggingNode(new UnionNode(unique, planAddedStatements1), "");


Expand All @@ -106,21 +99,18 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS

PlanNode planAddedForShape = new LoggingNode(nodeShape.getPlanAddedStatements(shaclSailConnection, nodeShape), "");

PlanNode select = new LoggingNode(shaclSailConnection.getCachedNodeFor(new Select(shaclSailConnection.getAddedStatements(), path.getQuery("?a", "?c", null))), "");
PlanNode addedByPath = new LoggingNode(getPlanAddedStatements(shaclSailConnection, nodeShape), "");

addedByPath = new LoggingNode((nodeShape).getTargetFilter(shaclSailConnection, addedByPath), "");

if (nodeShape instanceof TargetClass) {
select = new LoggingNode(((TargetClass) nodeShape).getTypeFilterPlan(shaclSailConnection, select), "");
}
topNode = new LoggingNode(new UnionNode(planAddedForShape, select), "");
topNode = new LoggingNode(new UnionNode(planAddedForShape, addedByPath), "");

}


PlanNode groupBy = new LoggingNode(new GroupByCount(topNode), "");

DirectTupleFromFilter filteredStatements = new DirectTupleFromFilter();
new MinCountFilter(groupBy, null, filteredStatements, minCount);
PlanNode filteredStatements = new MinCountFilter(groupBy, minCount).getFalseNode(UnBufferedPlanNode.class);

PlanNode minCountFilter = new LoggingNode(filteredStatements, "");

Expand All @@ -130,8 +120,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS

PlanNode groupBy2 = new LoggingNode(new GroupByCount(bulkedExternalLeftOuterJoin2), "");

DirectTupleFromFilter filteredStatements2 = new DirectTupleFromFilter();
new MinCountFilter(groupBy2, null, filteredStatements2, minCount);
PlanNode filteredStatements2 = new MinCountFilter(groupBy2, minCount).getFalseNode(UnBufferedPlanNode.class);

if (printPlans) {
String planAsGraphvizDot = getPlanAsGraphvizDot(filteredStatements2, shaclSailConnection);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new LiteralComparatorFilter(parent, trueNode, falseNode, minExclusive, value -> value < 0),
(parent) -> new LiteralComparatorFilter(parent, minExclusive, value -> value < 0),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new LiteralComparatorFilter(parent, trueNode, falseNode, minInclusive, value -> value <= 0),
(parent) -> new LiteralComparatorFilter(parent, minInclusive, value -> value <= 0),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new MinLengthFilter(parent, trueNode, falseNode, minLength),
(parent) -> new MinLengthFilter(parent, minLength),
this,
overrideTargetNode);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public PlanNode getPlan(ShaclSailConnection shaclSailConnection, NodeShape nodeS
PlanNode invalidValues = StandardisedPlanHelper.getGenericSingleObjectPlan(
shaclSailConnection,
nodeShape,
(parent, trueNode, falseNode) -> new NodeKindFilter(parent, trueNode, falseNode, nodeKind),
(parent) -> new NodeKindFilter(parent, nodeKind),
this,
overrideTargetNode);

Expand Down
Loading

0 comments on commit 7c81edb

Please sign in to comment.