Merge branch 'develop' into gh-3120-improve-matchedVertex-tests

library/tinkerpop/README.md

@@ -1,98 +1,15 @@
-Copyright 2016-2023 Crown Copyright
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
 # GafferPop
 
-GafferPop is a lightweight Gaffer implementation of TinkerPop, where TinkerPop methods are delegated to Gaffer graph operations.
-
-It is still in development. The implementation is basic and its performance is unknown in comparison to using Gaffer OperationChains.
-
-
-## Setup
-You must provide a GafferPop configuration file containing a path to a Gaffer store.properties file and comma separated list of of paths for Gaffer schema files, e.g:
-```
-gremlin.graph=uk.gov.gchq.gaffer.gaffer.tinkerpop.GafferPopGraph
-gaffer.storeproperties=conf/gaffer/map-store.properties
-gaffer.schemas=conf/gaffer/schema/
-```
-
-### Local example
-To run a demo local GafferPop graph, first download gremlin console: `apache-tinkerpop-gremlin-console-3.6.4.zip`
-
-To get going with the tinkerpop-modern dataset backed by a MapStore you can do the following:
-```bash
-# Build the code
-mvn clean install -Pquick -pl :tinkerpop -am
-
-gremlinConsolePath=[gremlin-console-path]
-
-# Create the necessary directories in the gremlin console folder
-mkdir -p $gremlinConsolePath/ext/gafferpop/plugin
-mkdir -p $gremlinConsolePath/conf/gafferpop
-
-# Copy the required files into the gremlin console folder
-cp -R tinkerpop/src/test/resources/* $gremlinConsolePath/conf/gafferpop
-cp -R tinkerpop/target/tinkerpop-*.jar tinkerpop/target/gafferpop-*.jar  $gremlinConsolePath/ext/gafferpop/plugin
-
-# Change from MiniAccumuloStore to MapStore properties
-sed -i 's/store.properties/map-store.properties/g' $gremlinConsolePath/conf/gafferpop/gafferpop-tinkerpop-modern.properties
-
-# Start gremlin
-cd $gremlinConsolePath
-./bin/gremlin.sh
-
-# Activate the GafferPop plugin
-:plugin use gafferpop
-```
-
-
-Within the gremlin console, load the tinkerpop modern data set:
-```
-graph = GraphFactory.open('conf/gafferpop/gafferpop-tinkerpop-modern.properties')
-graph.io(graphml()).readGraph('data/tinkerpop-modern.xml')
-g = graph.traversal()
-```
-do some queries:
-```
-g.V('1').hasLabel('person')
-g.V('1', '2').hasLabel('person').outE('knows').values().is(lt(1))
-```
-calculate the paths from 1 to 3 (max 6 loops):
-```
-start = '1';
-end = '3';
-g.V(start).repeat(bothE().otherV().simplePath()).until(hasId(end).or().loops().is(6)).path()
-```
-
-## Gaffer mapping to TinkerPop terms
-
- - Group -> Label
- - Vertex -> Vertex with label 'id'
- - Entity -> Vertex
- - Edge -> Edge
- - Edge ID -> [dest, source]
+GafferPop is a lightweight Gaffer implementation of the
+[TinkerPop framework](https://tinkerpop.apache.org/), where TinkerPop methods
+are delegated to Gaffer graph operations.
 
+It is still in development. The implementation is basic and its performance is
+unknown in comparison to using standard Gaffer `OperationChains`.
 
-## Limitations
+Please see the [official Gaffer documentation](https://gchq.github.io/gaffer-doc/latest/)
+for more information on the current GafferPop implementation, some useful links are as follows:
 
-There are several restrictions with this implementation. The following TinkerPop features are not implemented in GafferPop:
- - property index for unseeded queries (not yet implemented)
- - use of TraversalStrategy to speed up queries (not yet implemented)
- - suppling Gaffer user for auth (not yet implemented)
- - Removal of entities (Gaffer cannot do this)
- - Updating properties (Gaffer cannot do this)
- - Undirected edges (needs to be verified)
- - Entity group 'id' is reserved for an empty group containing only the vertex id (needs to be verified)
- - When you get the in or out Vertex directly off an Edge it will not contain any actual properties - it just returns the ID vertex. This is due to Gaffer allowing multiple entities to be associated with the source and destination vertices of an Edge. (needs to be verified)
+- [Set up a Gremlin console connection using GafferPop](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/).
+- [Basic Gremlin querying guide](https://gchq.github.io/gaffer-doc/latest/user-guide/query/gremlin/gremlin/).
+- [Current limitations of the GafferPop implementation](https://gchq.github.io/gaffer-doc/latest/user-guide/query/gremlin/gremlin-limits/).

library/tinkerpop/pom.xml

@@ -26,7 +26,12 @@
     <artifactId>tinkerpop</artifactId>
 
     <properties>
-        <tinkerpop.version>3.6.4</tinkerpop.version>
+        <tinkerpop.version>3.7.1</tinkerpop.version>
+        <google-guice.version>4.2.3</google-guice.version>
+        <cucumber.version>7.15.0</cucumber.version>
+
+        <!-- Don't run any of the standard tinkerpop tests by default-->
+        <test>!GafferPopGraphStructureStandardTest,!GafferPopFeatureTest</test>
     </properties>
 
     <dependencies>
@@ -90,6 +95,25 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <!-- For standard gremlin test suites -->
+        <dependency>
+            <groupId>org.apache.tinkerpop</groupId>
+            <artifactId>gremlin-test</artifactId>
+            <version>${tinkerpop.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-junit</artifactId>
+            <version>${cucumber.version}</version>
+            <scope>test</scope>
+        </dependency>
+         <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <version>${google-guice.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -115,4 +139,24 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>tinkerpop-standard-test</id>
+            <properties>
+                <!-- Gaffer doesn't pass all standard gremlin tests, this will hopefully improve with time -->
+                <maven.test.failure.ignore>true</maven.test.failure.ignore>
+                <test>GafferPopGraphStructureStandardTest,GafferPopFeatureTest</test>
+            </properties>
+            <dependencies>
+                <!-- Standard test suite still uses juint 4 so needs the vintage engine to run -->
+                <dependency>
+                    <groupId>org.junit.vintage</groupId>
+                    <artifactId>junit-vintage-engine</artifactId>
+                    <version>${junit.version}</version>
+                    <scope>test</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
 </project>

library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java

@@ -21,7 +21,10 @@
 import org.apache.tinkerpop.gremlin.structure.Property;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -45,56 +48,81 @@
  * </p>
  */
 public final class GafferPopEdge extends GafferPopElement implements Edge {
-    private Map<String, Property> properties;
-    private GafferPopVertex inVertex;
-    private GafferPopVertex outVertex;
+    private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopEdge.class);
 
-    public GafferPopEdge(final String label, final Object outVertexId, final Object inVertexId, final GafferPopGraph graph) {
-        this(label, checkVertex(outVertexId, graph), checkVertex(inVertexId, graph), graph);
-    }
+    private Map<String, Property> properties;
+    private final GafferPopVertex inVertex;
+    private final GafferPopVertex outVertex;
 
-    public GafferPopEdge(final String label, final GafferPopVertex outVertex, final GafferPopVertex inVertex, final GafferPopGraph graph) {
-        super(label, Arrays.asList(outVertex.id(), inVertex.id()), graph);
-        this.outVertex = checkVertex(outVertex, graph);
-        this.inVertex = checkVertex(inVertex, graph);
+    public GafferPopEdge(final String label, final Object outVertex, final Object inVertex, final GafferPopGraph graph) {
+        super(label, Arrays.asList(getVertexId(outVertex), getVertexId(inVertex)), graph);
+        this.outVertex = getValidVertex(outVertex, graph);
+        this.inVertex = getValidVertex(inVertex, graph);
     }
 
     @Override
     public <V> Property<V> property(final String key, final V value) {
         if (isReadOnly()) {
-            throw new UnsupportedOperationException("Updates are not supported");
-        }
-        ElementHelper.validateProperty(key, value);
-        final Property<V> newProperty = new GafferPopProperty<>(this, key, value);
-        if (null == this.properties) {
-            this.properties = new HashMap<>();
+            throw new UnsupportedOperationException("Updates are not supported, Edge is readonly");
         }
-        this.properties.put(key, newProperty);
+
+        // Attach properties before updating the graph
+        final Property<V> newProperty = propertyWithoutUpdate(key, value);
+        LOGGER.info("Updating Edge properties via aggregation");
+
+        // Re add to do the update via aggregation
+        graph().addEdge(this);
         return newProperty;
     }
 
     @Override
     public <V> Property<V> property(final String key) {
-        return null == this.properties ? Property.<V>empty() : this.properties.getOrDefault(key, Property.<V>empty());
+        return null == properties ? Property.<V>empty() : properties.getOrDefault(key, Property.<V>empty());
     }
 
     @Override
     public <V> Iterator<Property<V>> properties(final String... propertyKeys) {
-        if (null == this.properties) {
+        if (properties == null) {
             return Collections.emptyIterator();
         }
         if (propertyKeys.length == 1) {
-            final Property<V> property = this.properties.get(propertyKeys[0]);
+            final Property<V> property = properties.get(propertyKeys[0]);
             return null == property ? Collections.emptyIterator() : IteratorUtils.of(property);
         } else {
-            return (Iterator) this.properties.entrySet()
+            return (Iterator) properties.entrySet()
                     .stream()
                     .filter(entry -> ElementHelper.keyExists(entry.getKey(), propertyKeys))
                     .map(entry -> entry.getValue()).collect(Collectors.toList())
                     .iterator();
         }
     }
 
+
+    /**
+     * Updates the properties attached to this Edge but without modifying the
+     * underlying graph.
+     *
+     * This method is largely a helper for generating GafferPopEdge objects from
+     * Gaffer Edge returned from the graph. In this situation we want to be
+     * able to create a representative GafferPopEdge but without modifying the
+     * one stored in the graph.
+     *
+     * @param <V> Value type
+     * @param key The key
+     * @param value The value
+     * @return The property
+     */
+    public <V> Property<V> propertyWithoutUpdate(final String key, final V value) {
+        ElementHelper.validateProperty(key, value);
+        final Property<V> newProperty = new GafferPopProperty<>(this, key, value);
+        if (properties == null) {
+            properties = new HashMap<>();
+        }
+        properties.put(key, newProperty);
+
+        return newProperty;
+    }
+
     @Override
     public Iterator<Vertex> vertices(final Direction direction) {
         switch (direction) {
@@ -109,12 +137,18 @@ public Iterator<Vertex> vertices(final Direction direction) {
 
     @Override
     public Set<String> keys() {
-        return null == this.properties ? Collections.emptySet() : this.properties.keySet();
+        return properties == null ? Collections.emptySet() : properties.keySet();
+    }
+
+    @Override
+    public void remove() {
+        // Gaffer does not support deleting elements
+        throw Edge.Exceptions.edgeRemovalNotSupported();
     }
 
     @Override
     public String toString() {
-        return "e[" + outVertex().id() + "-" + label + "->" + inVertex().id() + "]";
+        return StringFactory.edgeString(this);
     }
 
     @Override
@@ -127,29 +161,62 @@ public Vertex inVertex() {
         return inVertex;
     }
 
-    private static GafferPopVertex checkVertex(final Object vertexId, final GafferPopGraph graph) {
-        final GafferPopVertex gafferPopVertex;
-        if (vertexId instanceof Vertex) {
-            if (vertexId instanceof GafferPopVertex) {
-                gafferPopVertex = checkVertex(((GafferPopVertex) vertexId), graph);
-            } else {
-                gafferPopVertex = new GafferPopVertex(GafferPopGraph.ID_LABEL, ((Vertex) vertexId).id(), graph);
-            }
+    /**
+     * Gets the vertex ID object from the supplied vertex.
+     *
+     * Will check if the supplied Object implements the {@link Vertex} interface
+     * if so will pull it from the instance otherwise assumes the Object itself
+     * is the ID.
+     *
+     * @param vertex The vertex Object or ID.
+     * @return The ID for the vertex.
+     */
+    private static Object getVertexId(final Object vertex) {
+        // Check if we need to pull the ID from the vertex or can use it directly
+        if (vertex instanceof Vertex) {
+            return ((GafferPopVertex) vertex).id();
         } else {
-            gafferPopVertex = new GafferPopVertex(GafferPopGraph.ID_LABEL, vertexId, graph);
+            return vertex;
         }
-
-        return gafferPopVertex;
     }
 
-    private static GafferPopVertex checkVertex(final GafferPopVertex gafferPopVertex, final GafferPopGraph graph) {
-        final GafferPopVertex gafferPopVertexId;
-        if (GafferPopGraph.ID_LABEL.equals(gafferPopVertex.label())) {
-            gafferPopVertexId = gafferPopVertex;
-        } else {
-            gafferPopVertexId = new GafferPopVertex(GafferPopGraph.ID_LABEL, gafferPopVertex.id(), graph);
+    /**
+     * Determines the GafferPopVertex object to connect this GafferPopEdge on.
+     *
+     * @param vertex The vertex object or ID
+     * @param graph The graph
+     * @return A valid GafferPopVertex based on the supplied object or ID.
+     */
+    private static GafferPopVertex getValidVertex(final Object vertex, final GafferPopGraph graph) {
+        // Determine if we can cast the vertex object we have been supplied
+        if (vertex instanceof Vertex) {
+            return (GafferPopVertex) vertex;
         }
 
-        return gafferPopVertexId;
+        // As a fallback assume its the vertex ID object and construct with the ID label.
+        return new GafferPopVertex(GafferPopGraph.ID_LABEL, vertex, graph);
+
+        /*
+         * TODO: Review whether a search should be carried out to determine the correct
+         *       Entity to use to construct the GafferPopVertex to add this edge too.
+         *       Currently a default label will be used if a vertex ID is given to this
+         *       method which may result in an incorrect mapping of a GafferPop
+         *       'label' to a Gaffer 'group'.
+         *
+         * A basic example of a search is given below:
+         *
+         * OperationChain<Iterable<? extends Element>> findBasedOnID = new OperationChain.Builder()
+         *     .first(new GetElements.Builder().input(new EntitySeed(vertex)).build())
+         *     .build();
+         *
+         * Iterable<? extends Element> result = graph.execute(findBasedOnID);
+         * Object foundEntity = StreamSupport.stream(result.spliterator(), false)
+         *     .filter(Entity.class::isInstance)
+         *     .findFirst()
+         *     .get();
+         *
+         * return new GafferPopVertexGenerator(graph)._apply((Element) foundEntity);
+         */
     }
+
 }