Skip to content

Commit

Permalink
Gh-3246: Add an explain step and GafferPop to REST API (#3248)
Browse files Browse the repository at this point in the history 
* basic rest controller for gremlin

* basic websocket endpoint for gremlin inside the rest api

* improvements to the gremlin ws endpoint

* basic hooks for storing last chain

* Add working explain endpoint and tidy websocket

* Add open telementry reporting

* overhaul gremlin user auth and move cypher translator

* basic testing for gremlin controller

* move to utils class

* basic testing updates

* update the options parsing and testing

* test tweaks

* updates from testing

* tweaks

* spotless

* address comments

---------

Co-authored-by: wb36499 <166839644+wb36499@users.noreply.github.com>
  • Loading branch information
@tb06904 @wb36499
tb06904 and wb36499 authored Jul 17, 2024
1 parent 717f544 commit 33e3fa7
Show file tree
Hide file tree
Showing 38 changed files with 1,170 additions and 597 deletions.
15 changes: 13 additions & 2 deletions library/tinkerpop/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
<artifactId>tinkerpop</artifactId>

<properties>
<tinkerpop.version>3.7.1</tinkerpop.version>
<google-guice.version>4.2.3</google-guice.version>
<cucumber.version>7.15.0</cucumber.version>
<!-- Note later versions require scala version 2.12.x -->
Expand Down Expand Up @@ -108,7 +107,7 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<dependency>
<groupId>uk.gov.gchq.gaffer</groupId>
<artifactId>federated-store</artifactId>
<version>${project.parent.version}</version>
Expand Down Expand Up @@ -156,6 +155,18 @@
</execution>
</executions>
</plugin>
<!-- Build the test jar so can be reused for testing the rest API -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down
23 changes: 15 additions & 8 deletions library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,12 @@ public enum HasStepFilterStage {
*/
public static final String NOT_READ_ONLY_ELEMENTS = "gaffer.elements.notreadonly";

/**
* Key for use in the store properties to allow setting the file location of
* the GafferPop properties file from a store properties file.
*/
public static final String GAFFERPOP_PROPERTIES = "gaffer.gafferpop.properties";

/**
* The vertex label for vertex IDs. These are {@link GafferPopVertex}s that
* don't have any properties, just an ID value and a label of 'id'.
Expand Down Expand Up @@ -707,15 +713,15 @@ public <T> T execute(final OperationChain<T> opChain) {
}
}
}
// Use the requested user based on variables
User user = new User.Builder()
.userId(variables.getUserId())
.dataAuths(variables.getDataAuths())
.build();

// Add the current chain to the list of chains ran so far for this query (it is reset by the graph step)
List<Operation> currentChain = variables.getLastOperationChain().getOperations();
currentChain.add(opChain);
variables.set(GafferPopGraphVariables.LAST_OPERATION_CHAIN, new OperationChain<>(currentChain));

try {
LOGGER.info("GafferPop operation chain called: {}", opChain.toOverviewString());
return graph.execute(opChain, user);
return graph.execute(opChain, variables.getUser());
} catch (final Exception e) {
LOGGER.error("Operation chain failed: {}", e.getMessage());
throw new RuntimeException("GafferPop operation failed: " + e.getMessage(), e);
Expand Down Expand Up @@ -989,13 +995,14 @@ private IncludeIncomingOutgoingType getInOutType(final Direction direction) {
* @param variables The variables
*/
public void setDefaultVariables(final GafferPopGraphVariables variables) {
LOGGER.info("Resetting graph variables to defaults");
variables.set(GafferPopGraphVariables.OP_OPTIONS, Collections.unmodifiableMap(opOptions));
variables.set(GafferPopGraphVariables.USER_ID, defaultUser.getUserId());
variables.set(GafferPopGraphVariables.DATA_AUTHS, configuration().getStringArray(DATA_AUTHS));
variables.set(GafferPopGraphVariables.USER, defaultUser);
variables.set(GafferPopGraphVariables.GET_ALL_ELEMENTS_LIMIT,
configuration().getInteger(GET_ALL_ELEMENTS_LIMIT, DEFAULT_GET_ALL_ELEMENTS_LIMIT));
variables.set(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE,
configuration().getString(HAS_STEP_FILTER_STAGE, DEFAULT_HAS_STEP_FILTER_STAGE.toString()));
variables.set(GafferPopGraphVariables.LAST_OPERATION_CHAIN, new OperationChain());
}

/**
Expand Down
57 changes: 30 additions & 27 deletions library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphVariables.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import uk.gov.gchq.gaffer.operation.OperationChain;
import uk.gov.gchq.gaffer.user.User;

import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
Expand All @@ -36,15 +39,20 @@ public final class GafferPopGraphVariables implements Graph.Variables {
public static final String OP_OPTIONS = "operationOptions";

/**
* Variable key for the list of data auths for the user interacting with the graph.
* Variable key for the list of data auths for the default user.
*/
public static final String DATA_AUTHS = "dataAuths";

/**
* Variable key for the userId of who is interacting with the graph.
* Variable key for the userId used for constructing a default user.
*/
public static final String USER_ID = "userId";

/**
* Variable key for the user who is interacting with the graph.
*/
public static final String USER = "user";

/**
* The max number of elements that can be returned by GetAllElements
*/
Expand All @@ -55,6 +63,16 @@ public final class GafferPopGraphVariables implements Graph.Variables {
*/
public static final String HAS_STEP_FILTER_STAGE = "hasStepFilterStage";

/**
* Key used in a with step to include a opencypher query traversal
*/
public static final String CYPHER_KEY = "cypher";

/**
* The variable with the last Gaffer operation chain that was ran from the Gremlin query
*/
public static final String LAST_OPERATION_CHAIN = "lastOperation";


private final Map<String, Object> variables;

Expand Down Expand Up @@ -91,17 +109,15 @@ public void set(final String key, final Object value) {
} else if (value instanceof Map) {
variables.put(key, value);
} else {
LOGGER.error(VAR_UPDATE_ERROR_STRING, OP_OPTIONS, value.getClass());
LOGGER.error(VAR_UPDATE_ERROR_STRING, key, value.getClass());
}
break;

case DATA_AUTHS:
if (value instanceof String[]) {
case USER:
if (value instanceof User) {
variables.put(key, value);
} else if (value instanceof String) {
variables.put(key, ((String) value).split(","));
} else {
LOGGER.error(VAR_UPDATE_ERROR_STRING, DATA_AUTHS, value.getClass());
LOGGER.error(VAR_UPDATE_ERROR_STRING, key, value.getClass());
}
break;

Expand Down Expand Up @@ -137,25 +153,8 @@ public Map<String, String> getOperationOptions() {
return new HashMap<>();
}

/**
* Gets the list of data auths.
*
* @return List of data auths.
*/
public String[] getDataAuths() {
if (variables.containsKey(DATA_AUTHS)) {
return (String[]) variables.get(DATA_AUTHS);
}
return new String[0];
}

/**
* Gets the current user ID.
*
* @return The user ID
*/
public String getUserId() {
return (String) variables.get(USER_ID);
public User getUser() {
return (User) variables.get(USER);
}

public Integer getAllElementsLimit() {
Expand All @@ -166,6 +165,10 @@ public String getHasStepFilterStage() {
return (String) variables.get(HAS_STEP_FILTER_STAGE);
}

public OperationChain<?> getLastOperationChain() {
return (OperationChain) variables.get(LAST_OPERATION_CHAIN);
}

public String toString() {
return StringFactory.graphVariablesString(this);
}
Expand Down
7 changes: 3 additions & 4 deletions ...src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,11 @@

/**
* Custom GafferPop GraphStep provides Gaffer specific optimisations
* for the initial GraphStep in a query. Also responsible for parsing
* any options passed via a 'with()' call on the query.
* for the initial GraphStep in a query.
* Also responsible for parsing any options passed via a 'with()' step
* on the query.
*
* <pre>
* g.with("userId", "user").V() // userId extracted to be used in the operation executions
* g.with("dataAuths", "write-access,read-access").V() // user access controls to apply on the user
* g.with("operationOptions", ["graphId:graph1", "opt1:val1"]).V() // operation options extracted and applied
* </pre>
*/
Expand Down
32 changes: 16 additions & 16 deletions .../gaffer/tinkerpop/process/traversal/strategy/optimisation/GafferPopGraphStepStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,18 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables;
import uk.gov.gchq.gaffer.tinkerpop.process.traversal.step.GafferPopGraphStep;

import java.util.Map;
import java.util.Optional;

/**
* The {@link GraphStep} strategy for GafferPop, this will replace the default
* {@link GraphStep} of a query to add Gaffer optimisations. Such as gathering
* any {@link HasStep}s so that a Gaffer View can be constructed for the query.
* Will also handle the translation any Cypher queries passed in a with() step
* into a Gremlin traversal.
* Will also handle the translation of Cypher queries passed via a with()
* step in the Gremlin traversal.
*
* <pre>
* g.V().hasLabel() // replaced by GafferPopGraphStep
Expand All @@ -52,27 +54,25 @@ public final class GafferPopGraphStepStrategy extends AbstractTraversalStrategy<
private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopGraphStepStrategy.class);
private static final GafferPopGraphStepStrategy INSTANCE = new GafferPopGraphStepStrategy();

/**
* Key used in a with step to include a opencypher query traversal
*/
public static final String CYPHER_KEY = "cypher";

private GafferPopGraphStepStrategy() {
}

@Override
public void apply(final Admin<?, ?> traversal) {
// Check for any options on the traversal
// Parse any options on the traversal
Optional<OptionsStrategy> optionsStrategy = traversal.getStrategies().getStrategy(OptionsStrategy.class);
// Translate and add a cypher traversal in if that key has been set
if (optionsStrategy.isPresent() && optionsStrategy.get().getOptions().containsKey(CYPHER_KEY)) {
LOGGER.info("Replacing traversal with translated Cypher query");
CypherAst ast = CypherAst.parse((String) optionsStrategy.get().getOptions().get(CYPHER_KEY));
Admin<?, ?> translatedCypher = ast.buildTranslation(Translator.builder().traversal().enableCypherExtensions().build()).asAdmin();
if (optionsStrategy.isPresent()) {
Map<String, Object> options = optionsStrategy.get().getOptions();
// Translate and add a cypher traversal in if that key has been set
if (options.containsKey(GafferPopGraphVariables.CYPHER_KEY)) {
LOGGER.info("Replacing traversal with translated Cypher query");
CypherAst ast = CypherAst.parse((String) options.get(GafferPopGraphVariables.CYPHER_KEY));
Admin<?, ?> translatedCypher = ast.buildTranslation(Translator.builder().traversal().enableCypherExtensions().build()).asAdmin();

// Add the cypher traversal
TraversalHelper.insertTraversal(0, translatedCypher, traversal);
LOGGER.debug("New traversal is: {}", traversal);
// Add the cypher traversal
TraversalHelper.insertTraversal(0, translatedCypher, traversal);
LOGGER.debug("New traversal is: {}", traversal);
}
}

TraversalHelper.getStepsOfClass(GraphStep.class, traversal).forEach(originalGraphStep -> {
Expand Down
Loading

0 comments on commit 33e3fa7

Please sign in to comment.