Merge branch 'develop' into feature/101_shacl_w3c_tests

* develop: (80 commits)
  eclipse-rdf4j/rdf4j#1298 performance enhancement
  eclipse-rdf4j/rdf4j#1298 made rdfs reasoner configurable and added back tests
  eclipse-rdf4j/rdf4j#1298 minor cleanup
  eclipse-rdf4j/rdf4j#1298 fix for dedupes
  eclipse-rdf4j/rdf4j#1298 initial implementation of a rdfs reasoner for the shacl engine
  eclipse-rdf4j/rdf4j#1297 implement requiresEvaluation
  eclipse-rdf4j/rdf4j#1297 change SPARQL order to improve performance
  eclipse-rdf4j/rdf4j#1297 update benchmarks
  eclipse-rdf4j/rdf4j#1111 hot fix caching broken after merging sh:class and also fix some tests
  hotfix benchmark
  eclipse-rdf4j/rdf4j#1111 optimizations for empty sail and bulk joining for previousStateConnection
  eclipse-rdf4j/rdf4j#1111 removed use of getPreviousStateConnection because it doesn't handle deleted, will need to think of something smarter here. Also fixed printing of plans.
  eclipse-rdf4j/rdf4j#1111 fixes for blank node support
  Moved test from testsuite to storage
  Fixes #1293 Allow Regex calls with non Var arguments be replaced with other string functions
  eclipse-rdf4j/rdf4j#1111 optimized imports
  eclipse-rdf4j/rdf4j#1111 fixed last edge case
  eclipse-rdf4j/rdf4j#1111 better support for OR
  eclipse-rdf4j/rdf4j#1111 fixed sh:or by doing a two pass for sh:or. Not quite done yet though.
  documentation and code cleanup
  ...

Signed-off-by: Håvard Ottestad <hmottestad@gmail.com>

# Conflicts:
#	shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/Utils.java

compliance/shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/SHACLComplianceTest.java

@@ -24,18 +24,20 @@
  */
 public class SHACLComplianceTest extends AbstractSHACLTest {
 
+	// set this to true to run all tests!
+	final static boolean RUN_ALL = false;
+
 	public static TestSuite suite()
 		throws Exception
 	{
+		String[] ignoredDirectories = {"targets", "sparql", "complex", "misc", "node", "path", "validation-reports", "property"};
+		if(RUN_ALL) ignoredDirectories = new String[0];
+
 		return new SHACLManifestTestSuiteFactory().createTestSuite(new TestFactory() {
 
 			@Override
 			public AbstractSHACLTest createSHACLTest(String testURI, String label, Model shapesGraph,
-					Model dataGraph, boolean failure, boolean conforms)
-			{
-				if (label.contains("multiple") || label.contains("implicit")
-						|| label.contains("targetObjects"))
-					return null; // skip
+													 Model dataGraph, boolean failure, boolean conforms) {
 				return new SHACLComplianceTest(testURI, label, shapesGraph, dataGraph, failure, conforms);
 			}
 
@@ -44,7 +46,7 @@ public String getName() {
 				return SHACLComplianceTest.class.getName();
 			}
 
-		}, true, true, false, "sparql", "complex", "misc", "node", "path", "validation-reports", "property");
+		}, true, true, false, ignoredDirectories);
 	}
 
 	public SHACLComplianceTest(String testURI, String label, Model shapesGraph, Model dataGraph,
@@ -59,7 +61,9 @@ protected NotifyingSail newDataSail() {
 
 	@Override
 	protected Sail newSail() {
-		return new ShaclSail(newDataSail());
+		ShaclSail shaclSail = new ShaclSail(new MemoryStore());
+		shaclSail.setIgnoreNoShapesLoadedException(true);
+		return shaclSail;
 	}
 
 }

compliance/solr/solr/cores/embedded/conf/managed-schema

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Solr managed schema - automatically generated - DO NOT EDIT -->
+<schema name="LuceneSail" version="1.5">
+  <uniqueKey>id</uniqueKey>
+  <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
+  <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+  <fieldType name="geo" class="solr.SpatialRecursivePrefixTreeFieldType" indexed="true" stored="true" distanceUnits="kilometers"/>
+  <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
+  <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+  <fieldType name="string" class="solr.StrField" omitNorms="true" omitTermFreqAndPositions="true" indexed="true" stored="true" sortMissingLast="true" docValues="true" multiValued="false"/>
+  <fieldType name="text" class="solr.TextField" omitNorms="false" omitTermFreqAndPositions="false" indexed="true" stored="true" positionIncrementGap="100" multiValued="true">
+    <analyzer type="index">
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+    </analyzer>
+    <analyzer type="query">
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+    </analyzer>
+  </fieldType>
+  <field name="_version_" type="long" indexed="true" stored="true"/>
+  <field name="context" type="string" default="null" required="false"/>
+  <field name="http://www.opengis.net/ont/geosparql#asWKT" type="geo" required="false"/>
+  <field name="id" type="string" required="true"/>
+  <field name="text" type="text"/>
+  <field name="uri" type="string" required="true"/>
+  <dynamicField name="*" type="text"/>
+</schema>

compliance/solr/solr/cores/embedded/conf/schema.xml.bak

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<schema name="LuceneSail" version="1.5">
+	<!-- Valid attributes for fields: name: mandatory - the name for the field 
+		type: mandatory - the name of a field type from the <types> fieldType section 
+		indexed: true if this field should be indexed (searchable or sortable) stored: 
+		true if this field should be retrievable docValues: true if this field should 
+		have doc values. Doc values are useful for faceting, grouping, sorting and 
+		function queries. Although not required, doc values will make the index faster 
+		to load, more NRT-friendly and more memory-efficient. They however come with 
+		some limitations: they are currently only supported by StrField, UUIDField 
+		and all Trie*Fields, and depending on the field type, they might require 
+		the field to be single-valued, be required or have a default value (check 
+		the documentation of the field type you're interested in for more information) 
+		multiValued: true if this field may contain multiple values per document 
+		omitNorms: (expert) set to true to omit the norms associated with this field 
+		(this disables length normalization and index-time boosting for the field, 
+		and saves some memory). Only full-text fields or fields that need an index-time 
+		boost need norms. Norms are omitted for primitive (non-analyzed) types by 
+		default. termVectors: [false] set to true to store the term vector for a 
+		given field. When using MoreLikeThis, fields used for similarity should be 
+		stored for best performance. termPositions: Store position information with 
+		the term vector. This will increase storage costs. termOffsets: Store offset 
+		information with the term vector. This will increase storage costs. required: 
+		The field is required. It will throw an error if the value does not exist 
+		default: a value that should be used if no value is specified when adding 
+		a document. -->
+
+	<field name="_version_" type="long" indexed="true" stored="true" />
+
+	<field name="id" type="string" required="true" />
+	<field name="uri" type="string" required="true" />
+	<field name="context" type="string" required="false"
+		default="null" />
+	<field name="text" type="text" />
+	<field name="http://www.opengis.net/ont/geosparql#asWKT"
+		type="geo" required="false" />
+
+	<dynamicField name="*" type="text" />
+
+	<uniqueKey>id</uniqueKey>
+
+	<fieldType name="string" class="solr.StrField" indexed="true"
+		docValues="true" stored="true" multiValued="false" omitNorms="true"
+		omitTermFreqAndPositions="true" sortMissingLast="true" />
+
+	<fieldType name="text" class="solr.TextField" indexed="true"
+		stored="true" multiValued="true" omitNorms="false"
+		omitTermFreqAndPositions="false" positionIncrementGap="100">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+			<!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" 
+				/> -->
+			<filter class="solr.LowerCaseFilterFactory" />
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+			<!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" 
+				/> <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" 
+				expand="true"/> -->
+			<filter class="solr.LowerCaseFilterFactory" />
+		</analyzer>
+	</fieldType>
+
+	<fieldType name="geo"
+		class="solr.SpatialRecursivePrefixTreeFieldType" indexed="true"
+		stored="true" distanceUnits="kilometers" />
+
+	<!-- <fieldType name="geo_jts" class="solr.SpatialRecursivePrefixTreeFieldType" 
+		indexed="true" stored="true" distanceUnits="kilometers" spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory"/> -->
+
+	<fieldType name="int" class="solr.TrieIntField"
+		precisionStep="0" positionIncrementGap="0" />
+	<fieldType name="float" class="solr.TrieFloatField"
+		precisionStep="0" positionIncrementGap="0" />
+	<fieldType name="long" class="solr.TrieLongField"
+		precisionStep="0" positionIncrementGap="0" />
+	<fieldType name="double" class="solr.TrieDoubleField"
+		precisionStep="0" positionIncrementGap="0" />
+</schema>

compliance/store/src/test/java/org/eclipse/rdf4j/repository/sail/memory/ShaclRepositoryConnectionTest.java

@@ -0,0 +1,32 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Eclipse RDF4J contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *******************************************************************************/
+
+package org.eclipse.rdf4j.repository.sail.memory;
+
+import org.eclipse.rdf4j.IsolationLevel;
+import org.eclipse.rdf4j.repository.Repository;
+import org.eclipse.rdf4j.repository.RepositoryConnectionTest;
+import org.eclipse.rdf4j.repository.RepositoryTest;
+import org.eclipse.rdf4j.repository.sail.SailRepository;
+import org.eclipse.rdf4j.sail.memory.MemoryStore;
+import org.eclipse.rdf4j.sail.shacl.ShaclSail;
+
+public class ShaclRepositoryConnectionTest extends RepositoryConnectionTest {
+
+	public ShaclRepositoryConnectionTest(IsolationLevel level) {
+		super(level);
+	}
+
+	@Override
+	protected Repository createRepository() {
+		ShaclSail shaclSail = new ShaclSail(new MemoryStore());
+		shaclSail.setIgnoreNoShapesLoadedException(true);
+		return new SailRepository(shaclSail);
+	}
+
+}

compliance/store/src/test/java/org/eclipse/rdf4j/repository/sail/memory/ShaclRepositoryTest.java

@@ -0,0 +1,26 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Eclipse RDF4J contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *******************************************************************************/
+
+package org.eclipse.rdf4j.repository.sail.memory;
+
+import org.eclipse.rdf4j.repository.Repository;
+import org.eclipse.rdf4j.repository.RepositoryTest;
+import org.eclipse.rdf4j.repository.sail.SailRepository;
+import org.eclipse.rdf4j.sail.memory.MemoryStore;
+import org.eclipse.rdf4j.sail.shacl.ShaclSail;
+
+public class ShaclRepositoryTest extends RepositoryTest {
+
+	@Override
+	protected Repository createRepository() {
+		ShaclSail shaclSail = new ShaclSail(new MemoryStore());
+		shaclSail.setIgnoreNoShapesLoadedException(true);
+
+		return new SailRepository(shaclSail);
+	}
+}

compliance/store/src/test/java/org/eclipse/rdf4j/sail/memory/ShaclConcurrencyTest.java

@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Eclipse RDF4J contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *******************************************************************************/
+
+package org.eclipse.rdf4j.sail.memory;
+
+import org.eclipse.rdf4j.sail.Sail;
+import org.eclipse.rdf4j.sail.SailConcurrencyTest;
+import org.eclipse.rdf4j.sail.SailException;
+import org.eclipse.rdf4j.sail.shacl.ShaclSail;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class ShaclConcurrencyTest extends SailConcurrencyTest {
+
+	/*---------*
+	 * Methods *
+	 *---------*/
+
+	@Override
+	protected Sail createSail() throws SailException {
+		ShaclSail shaclSail = new ShaclSail(new MemoryStore());
+		shaclSail.setIgnoreNoShapesLoadedException(true);
+		return shaclSail;
+	}
+
+	@Ignore
+	@Test
+	@Override
+	public void testConcurrentAddLargeTxnRollback() throws Exception {
+		// empty since this test is ignored
+	}
+}

compliance/store/src/test/java/org/eclipse/rdf4j/sail/shacl/ShaclIsolationLevelTest.java

@@ -0,0 +1,28 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Eclipse RDF4J contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *******************************************************************************/
+
+package org.eclipse.rdf4j.sail.shacl;
+
+import org.eclipse.rdf4j.sail.Sail;
+import org.eclipse.rdf4j.sail.SailException;
+import org.eclipse.rdf4j.sail.SailIsolationLevelTest;
+import org.eclipse.rdf4j.sail.memory.MemoryStore;
+
+public class ShaclIsolationLevelTest extends SailIsolationLevelTest {
+
+	/*---------*
+	 * Methods *
+	 *---------*/
+
+	@Override
+	protected Sail createSail() throws SailException {
+		ShaclSail shaclSail = new ShaclSail(new MemoryStore());
+		shaclSail.setIgnoreNoShapesLoadedException(true);
+		return shaclSail;
+	}
+}

evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/function/datetime/Seconds.java

@@ -7,6 +7,7 @@
  *******************************************************************************/
 package org.eclipse.rdf4j.query.algebra.evaluation.function.datetime;
 
+import java.math.BigDecimal;
 import javax.xml.datatype.DatatypeConstants;
 import javax.xml.datatype.XMLGregorianCalendar;
 
@@ -53,7 +54,11 @@ public Literal evaluate(ValueFactory valueFactory, Value... args)
 
 					int seconds = calValue.getSecond();
 					if (DatatypeConstants.FIELD_UNDEFINED != seconds) {
-						return valueFactory.createLiteral(String.valueOf(seconds), XMLSchema.DECIMAL);
+						BigDecimal fraction = calValue.getFractionalSecond();
+						String str = (fraction == null) ? String.valueOf(seconds) 
+														: String.valueOf(fraction.doubleValue() + seconds);
+
+						return valueFactory.createLiteral(str, XMLSchema.DECIMAL);
 					}
 					else {
 						throw new ValueExprEvaluationException(

evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/function/hash/HashFunction.java

@@ -8,6 +8,7 @@
 package org.eclipse.rdf4j.query.algebra.evaluation.function.hash;
 
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
@@ -34,7 +35,7 @@ public abstract class HashFunction implements Function {
 	 */
 	protected String hash(String text, String algorithm)
 			throws NoSuchAlgorithmException {
-		byte[] hash = MessageDigest.getInstance(algorithm).digest(text.getBytes());
+		byte[] hash = MessageDigest.getInstance(algorithm).digest(text.getBytes(StandardCharsets.UTF_8));
 		BigInteger bi = new BigInteger(1, hash);
 
 		return String.format("%0" + hash.length * 2 + "x", bi);

evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/RegexAsStringFunctionOptimizer.java

@@ -51,7 +51,7 @@ public void meet(Regex node) {
             final ValueExpr flagsArg = node.getFlagsArg();
             if (flagsArg == null || flagsArg.toString().isEmpty()) {
                 //if we have no flags then we can not be in case insensitive mode
-                if (node.getPatternArg() instanceof ValueConstant && node.getArg() instanceof Var) {
+                if (node.getPatternArg() instanceof ValueConstant) {
                     ValueConstant vc = (ValueConstant) node.getPatternArg();
                     String regex = vc.getValue().stringValue();
                     final boolean anchoredAtStart = regex.startsWith("^");

evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/RegexAsStringFunctionOptimizierTest.java

@@ -88,6 +88,16 @@ public void testNotContains()
 
         testOptimizer(unoptimizedQuery, unoptimizedQuery);
     }
+
+    @Test
+    public void testContainsFunction()
+            throws MalformedQueryException {
+        String optimizedQuery = "SELECT ?o WHERE {?s ?p ?o . FILTER(CONTAINS(STR(?o), 'a'))}";
+        String unoptimizedQuery = "SELECT ?o WHERE {?s ?p ?o . FILTER(REGEX(STR(?o), 'a'))}";
+
+
+        testOptimizer(optimizedQuery, unoptimizedQuery);
+    }
 
     @Test
     public void testRealRegexDoesNotRedirect() {

memory/src/main/java/org/eclipse/rdf4j/sail/memory/MemoryStoreConnection.java

@@ -128,4 +128,8 @@ public void clearInferred(Resource... contexts)
 		super.clearInferred(contexts);
 		sailChangedEvent.setStatementsRemoved(true);
 	}
+
+	public MemoryStore getSail() {
+		return sail;
+	}
 }

nativerdf/pom.xml

@@ -61,6 +61,13 @@
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>rdf4j-repository-sail</artifactId>
+			<version>${project.version}</version>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>