From fae7e9a4cb680c52860eaeffd96370dcca98200f Mon Sep 17 00:00:00 2001 From: ntisseyre Date: Sun, 17 Mar 2024 21:36:49 -0500 Subject: [PATCH] Lazy load relations and cached query optimization Signed-off-by: ntisseyre --- docs/basics/transactions.md | 3 + .../janusgraph/graphdb/JanusGraphTest.java | 8 +- .../graphdb/database/LazyLoadGraphTest.java | 66 +++++ .../janusgraph/core/JanusGraphLazyEdge.java | 55 ++++ .../core/JanusGraphLazyRelation.java | 264 ++++++++++++++++++ .../JanusGraphLazyRelationConstructor.java | 66 +++++ .../core/JanusGraphLazyVertexProperty.java | 61 ++++ .../janusgraph/core/TransactionBuilder.java | 9 + .../graphdb/database/EdgeSerializer.java | 6 + .../query/condition/DirectionCondition.java | 14 +- .../BasicVertexCentricQueryBuilder.java | 57 ++++ .../MultiVertexCentricQueryBuilder.java | 6 +- .../transaction/RelationConstructor.java | 20 +- .../transaction/StandardJanusGraphTx.java | 22 ++ .../StandardTransactionBuilder.java | 24 +- .../transaction/TransactionConfiguration.java | 6 + .../graphdb/vertices/CacheVertex.java | 4 + .../graphdb/vertices/PreloadedVertex.java | 4 - .../inmemory/InMemoryLazyLoadGraphTest.java | 53 ++++ 19 files changed, 730 insertions(+), 18 deletions(-) create mode 100644 janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/database/LazyLoadGraphTest.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyEdge.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelation.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelationConstructor.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyVertexProperty.java create mode 100644 janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java diff --git a/docs/basics/transactions.md b/docs/basics/transactions.md index 46c1a1eeec0..91b3c8da50d 100644 --- a/docs/basics/transactions.md +++ b/docs/basics/transactions.md @@ -326,6 +326,9 @@ following aspects of a transaction to be configured: cache access during read operations. Doesn't have any effect if database level cache was disabled via config `cache.db-cache`. +- `lazyLoadRelations()` - Set lazy-load for all properties and edges of the vertex: ids and values are deserialized upon demand. + When enabled, it can have a boost on large-scale read operations, if only certain types of relations are being read from the vertex. + Once, the desired configuration options have been specified, the new transaction is started via `start()` which returns a `JanusGraphTransaction`. diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java index 4899b747930..19cd0c3e8e8 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java @@ -116,6 +116,7 @@ import org.janusgraph.graphdb.database.util.StaleIndexRecordUtil; import org.janusgraph.graphdb.internal.ElementCategory; import org.janusgraph.graphdb.internal.ElementLifeCycle; +import org.janusgraph.graphdb.internal.InternalElement; import org.janusgraph.graphdb.internal.InternalRelationType; import org.janusgraph.graphdb.internal.InternalVertex; import org.janusgraph.graphdb.internal.Order; @@ -129,7 +130,6 @@ import org.janusgraph.graphdb.query.condition.PredicateCondition; import org.janusgraph.graphdb.query.index.IndexSelectionUtil; import org.janusgraph.graphdb.query.profile.QueryProfiler; -import org.janusgraph.graphdb.relations.AbstractEdge; import org.janusgraph.graphdb.relations.RelationIdentifier; import org.janusgraph.graphdb.relations.StandardEdge; import org.janusgraph.graphdb.relations.StandardVertexProperty; @@ -425,7 +425,7 @@ public void testUpdatePropertyPropThenRemoveVertex() { public void testUpdateEdgePropertyThenRemoveEdge() { initializeGraphWithVerticesAndEdges(); // normal edge - AbstractEdge edge = (AbstractEdge) graph.traversal().E().has("_e", 1).next(); + InternalElement edge = (InternalElement) graph.traversal().E().has("_e", 1).next(); assertTrue(ElementLifeCycle.isLoaded(edge.getLifeCycle())); Object id = edge.id(); @@ -452,7 +452,7 @@ public void testUpdateEdgePropertyThenRemoveEdge() { public void testUpdateForkEdgePropertyThenRemoveEdge() { initializeGraphWithVerticesAndEdges(); // fork edge - AbstractEdge edge = (AbstractEdge) graph.traversal().E().has("_e", 2).next(); + InternalElement edge = (InternalElement) graph.traversal().E().has("_e", 2).next(); assertTrue(ElementLifeCycle.isLoaded(edge.getLifeCycle())); Object id = edge.id(); @@ -483,7 +483,7 @@ public void testUpdateForkEdgePropertyThenRemoveEdge() { @Test public void testUpdateForkEdgePropertyThenFindEdgeById() { initializeGraphWithVerticesAndEdges(); - AbstractEdge edge = (AbstractEdge) graph.traversal().E().has("_e", 2).next(); + InternalElement edge = (InternalElement) graph.traversal().E().has("_e", 2).next(); Object id = edge.id(); edge.property("_e", -2); diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/database/LazyLoadGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/database/LazyLoadGraphTest.java new file mode 100644 index 00000000000..248038a7d35 --- /dev/null +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/database/LazyLoadGraphTest.java @@ -0,0 +1,66 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.database; + +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexFilterOptimizerStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasIdOptimizerStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasUniquePropertyOptimizerStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexIsOptimizerStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphHasStepStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphIoRegistrationStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphLocalQueryOptimizerStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexAggStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMultiQueryStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphStepStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphUnusedMultiQueryRemovalStrategy; +import org.janusgraph.graphdb.transaction.StandardTransactionBuilder; + +public class LazyLoadGraphTest extends StandardJanusGraph { + + static { + TraversalStrategies graphStrategies = + TraversalStrategies.GlobalCache.getStrategies(Graph.class) + .clone() + .addStrategies(AdjacentVertexFilterOptimizerStrategy.instance(), + AdjacentVertexHasIdOptimizerStrategy.instance(), + AdjacentVertexIsOptimizerStrategy.instance(), + AdjacentVertexHasUniquePropertyOptimizerStrategy.instance(), + JanusGraphLocalQueryOptimizerStrategy.instance(), + JanusGraphHasStepStrategy.instance(), + JanusGraphMultiQueryStrategy.instance(), + JanusGraphUnusedMultiQueryRemovalStrategy.instance(), + JanusGraphMixedIndexAggStrategy.instance(), + JanusGraphMixedIndexCountStrategy.instance(), + JanusGraphStepStrategy.instance(), + JanusGraphIoRegistrationStrategy.instance()); + + //Register with cache + TraversalStrategies.GlobalCache.registerStrategies(LazyLoadGraphTest.class, graphStrategies); + } + + public LazyLoadGraphTest(GraphDatabaseConfiguration configuration) { + super(configuration); + } + + @Override + public StandardTransactionBuilder buildTransaction() { + return (StandardTransactionBuilder) new StandardTransactionBuilder(getConfiguration(), this) + .lazyLoadRelations(); + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyEdge.java b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyEdge.java new file mode 100644 index 00000000000..4d315f09755 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyEdge.java @@ -0,0 +1,55 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.core; + +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.diskstorage.Entry; +import org.janusgraph.graphdb.internal.InternalRelation; +import org.janusgraph.graphdb.internal.InternalRelationType; +import org.janusgraph.graphdb.internal.InternalVertex; +import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; + +public class JanusGraphLazyEdge extends JanusGraphLazyRelation implements JanusGraphEdge { + + public JanusGraphLazyEdge(InternalRelation janusGraphRelation, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + super(janusGraphRelation, vertex, tx, type); + } + + public JanusGraphLazyEdge(Entry dataEntry, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + super(dataEntry, vertex, tx, type); + } + + private JanusGraphEdge loadEdge() { + assert this.isEdge(); + return (JanusGraphEdge) this.loadValue(); + } + + @Override + public JanusGraphVertex vertex(Direction dir) { + return this.loadEdge().vertex(dir); + } + + @Override + public JanusGraphVertex otherVertex(Vertex vertex) { + return this.loadEdge().otherVertex(vertex); + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelation.java b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelation.java new file mode 100644 index 00000000000..943dc7fef0f --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelation.java @@ -0,0 +1,264 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.core; + +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.diskstorage.Entry; +import org.janusgraph.graphdb.internal.InternalRelation; +import org.janusgraph.graphdb.internal.InternalRelationType; +import org.janusgraph.graphdb.internal.InternalVertex; +import org.janusgraph.graphdb.transaction.RelationConstructor; +import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; + +import java.util.Iterator; + +public abstract class JanusGraphLazyRelation implements InternalRelation { + + private InternalRelation lazyLoadedRelation; + private Entry dataEntry; + private final InternalVertex vertex; + private final StandardJanusGraphTx tx; + private final InternalRelationType type; + + private final Object lockObject; + + public JanusGraphLazyRelation(InternalRelation janusGraphRelation, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + this.lazyLoadedRelation = janusGraphRelation; + this.dataEntry = null; + this.vertex = vertex; + this.tx = tx; + this.type = type; + this.lockObject = new Object(); + } + + public JanusGraphLazyRelation(Entry dataEntry, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + this.lazyLoadedRelation = null; + this.dataEntry = dataEntry; + this.vertex = vertex; + this.tx = tx; + this.type = type; + this.lockObject = new Object(); + } + + public InternalRelation loadValue() { + if (this.lazyLoadedRelation == null) { + synchronized (lockObject) { + if (this.lazyLoadedRelation == null) { + if (tx.isClosed()) { + throw new IllegalStateException("Any lazy load operation is not supported when transaction is already closed."); + } + //parseHeaderOnly = false: it doesn't give much latency on read, but gives a good boost on writes + boolean parseHeaderOnly = tx.getConfiguration().isReadOnly(); + this.lazyLoadedRelation = RelationConstructor.readRelation(this.vertex, this.dataEntry, parseHeaderOnly, this.tx); + } + } + } + return this.lazyLoadedRelation; + } + + public V value() { + assert this.isProperty(); + return ((Property) this.loadValue()).value(); + } + + @Override + public V value(String s) { + return this.loadValue().value(s); + } + + public boolean isSingle() { + return this.type.multiplicity().getCardinality().equals(Cardinality.SINGLE); + } + + @Override + public RelationType getType() { + return this.type; + } + + @Override + public Direction direction(Vertex vertex) { + return this.loadValue().direction(vertex); + } + + @Override + public boolean isIncidentOn(Vertex vertex) { + return this.loadValue().isIncidentOn(vertex); + } + + @Override + public boolean isLoop() { + return this.loadValue().isLoop(); + } + + @Override + public boolean isProperty() { + return this.type.isPropertyKey(); + } + + @Override + public boolean isEdge() { + return this.type.isEdgeLabel(); + } + + @Override + public String label() { + return this.loadValue().label(); + } + + @Override + public StandardJanusGraphTx tx() { + return this.loadValue().tx(); + } + + @Override + public JanusGraphTransaction graph() { + return this.vertex.graph(); + } + + @Override + public void setId(Object id) { + this.loadValue().setId(id); + } + + @Override + public Object id() { + return this.loadValue().id(); + } + + @Override + public byte getLifeCycle() { + return this.loadValue().getLifeCycle(); + } + + @Override + public boolean isInvisible() { + return this.type.isInvisibleType(); + } + + @Override + public long longId() { + return this.loadValue().longId(); + } + + @Override + public boolean hasId() { + return this.loadValue().hasId(); + } + + @Override + public void remove() { + this.loadValue().remove(); + } + + @Override + public Iterator> properties(String... propertyKeys) { + return this.loadValue().properties(propertyKeys); + } + + @Override + public Property property(String s, V v) { + return this.loadValue().property(s, v); + } + + @Override + public V valueOrNull(PropertyKey propertyKey) { + return this.loadValue().valueOrNull(propertyKey); + } + + @Override + public boolean isNew() { + return this.loadValue().isNew(); + } + + @Override + public boolean isLoaded() { + return this.loadValue().isLoaded(); + } + + @Override + public boolean isRemoved() { + return this.loadValue().isRemoved(); + } + + @Override + public InternalRelation it() { + InternalRelation loaded = this.loadValue(); + InternalRelation itRelation = loaded.it(); + if (itRelation == loaded) { + return this; + } else { + return JanusGraphLazyRelationConstructor.create(itRelation, vertex, tx); + } + } + + @Override + public InternalVertex getVertex(int pos) { + return this.loadValue().getVertex(pos); + } + + @Override + public int getArity() { + return this.loadValue().getArity(); + } + + @Override + public int getLen() { + return this.loadValue().getLen(); + } + + @Override + public O getValueDirect(PropertyKey key) { + return this.loadValue().getValueDirect(key); + } + + @Override + public void setPropertyDirect(PropertyKey key, Object value) { + this.loadValue().setPropertyDirect(key, value); + } + + @Override + public Iterable getPropertyKeysDirect() { + return this.loadValue().getPropertyKeysDirect(); + } + + @Override + public O removePropertyDirect(PropertyKey key) { + return this.loadValue().removePropertyDirect(key); + } + + @Override + public int hashCode() { + return this.loadValue().hashCode(); + } + + @Override + public boolean equals(Object other) { + InternalRelation thisRel = this.loadValue(); + if (other instanceof JanusGraphLazyRelation) { + InternalRelation otherRel = ((JanusGraphLazyRelation) other).loadValue(); + return thisRel.equals(otherRel); + } else { + return thisRel.equals(other); + } + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelationConstructor.java b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelationConstructor.java new file mode 100644 index 00000000000..b8c5eb93590 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyRelationConstructor.java @@ -0,0 +1,66 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.core; + +import org.janusgraph.diskstorage.Entry; +import org.janusgraph.graphdb.internal.InternalRelation; +import org.janusgraph.graphdb.internal.InternalRelationType; +import org.janusgraph.graphdb.internal.InternalVertex; +import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; +import org.janusgraph.graphdb.types.system.BaseRelationType; +import org.janusgraph.graphdb.types.system.SystemRelationType; + +public class JanusGraphLazyRelationConstructor { + + public static InternalRelation create(InternalRelation janusGraphRelation, + final InternalVertex vertex, + final StandardJanusGraphTx tx) { + long typeId = janusGraphRelation.getType().longId(); + InternalRelationType type = tx.getOrLoadRelationTypeById(typeId); + JanusGraphLazyRelation lazyRelation; + if (type.isPropertyKey()) { + lazyRelation = new JanusGraphLazyVertexProperty(janusGraphRelation, vertex, tx, type); + } else { + lazyRelation = new JanusGraphLazyEdge(janusGraphRelation, vertex, tx, type); + } + + if (type instanceof BaseRelationType) { + return lazyRelation.loadValue(); + } else { + return lazyRelation; + } + } + + public static InternalRelation create(Entry dataEntry, + final InternalVertex vertex, + final StandardJanusGraphTx tx) { + + long typeId = tx.getEdgeSerializer().parseTypeId(dataEntry); + InternalRelationType type = tx.getOrLoadRelationTypeById(typeId); + + JanusGraphLazyRelation lazyRelation; + if (type.isPropertyKey()) { + lazyRelation = new JanusGraphLazyVertexProperty(dataEntry, vertex, tx, type); + } else { + lazyRelation = new JanusGraphLazyEdge(dataEntry, vertex, tx, type); + } + + if (type instanceof SystemRelationType) { + return lazyRelation.loadValue(); + } else { + return lazyRelation; + } + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyVertexProperty.java b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyVertexProperty.java new file mode 100644 index 00000000000..8fb2baff1a0 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphLazyVertexProperty.java @@ -0,0 +1,61 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.core; + +import org.apache.tinkerpop.gremlin.structure.Property; +import org.janusgraph.diskstorage.Entry; +import org.janusgraph.graphdb.internal.InternalRelation; +import org.janusgraph.graphdb.internal.InternalRelationType; +import org.janusgraph.graphdb.internal.InternalVertex; +import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; + +import java.util.Iterator; + +public class JanusGraphLazyVertexProperty extends JanusGraphLazyRelation implements JanusGraphVertexProperty { + + public JanusGraphLazyVertexProperty(InternalRelation janusGraphRelation, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + super(janusGraphRelation, vertex, tx, type); + } + + public JanusGraphLazyVertexProperty(Entry dataEntry, + final InternalVertex vertex, + final StandardJanusGraphTx tx, + final InternalRelationType type) { + super(dataEntry, vertex, tx, type); + } + + private JanusGraphLazyVertexProperty loadProperty() { + assert this.isProperty(); + return (JanusGraphLazyVertexProperty) this.loadValue(); + } + + @Override + public boolean isPresent() { + return loadProperty().isPresent(); + } + + @Override + public JanusGraphVertex element() { + return loadProperty().element(); + } + + @Override + public Iterator> properties(String... propertyKeys) { + return this.loadProperty().properties(propertyKeys); + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java index 697de011f5a..6fb041070fa 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java @@ -150,6 +150,15 @@ public interface TransactionBuilder { */ TransactionBuilder skipDBCacheRead(); + /** + * Set lazy-load for all properties and edges: ids and values are deserialized upon demand. + *

+ * When enabled, it can have a boost on large scale read operations, when only certain type of relations are being read. + * + * @return Object with lazy-load setting. + */ + TransactionBuilder lazyLoadRelations(); + /** * Sets `has` step strategy mode. *

diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/EdgeSerializer.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/EdgeSerializer.java index 10ba1c8507f..b03356b9e01 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/EdgeSerializer.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/EdgeSerializer.java @@ -92,6 +92,12 @@ public Direction parseDirection(Entry data) { return IDHandler.readRelationType(data.asReadBuffer()).dirID.getDirection(); } + public long parseTypeId(Entry data) { + RelationCache map = data.getCache(); + if (map != null) return map.typeId; + return IDHandler.readRelationType(data.asReadBuffer()).typeId; + } + @Override public RelationCache parseRelation(Entry data, boolean excludeProperties, TypeInspector tx) { ReadBuffer in = data.asReadBuffer(); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/DirectionCondition.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/DirectionCondition.java index fa8ce3b7b0c..bffff9b7c2f 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/DirectionCondition.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/condition/DirectionCondition.java @@ -16,6 +16,7 @@ import org.apache.tinkerpop.gremlin.structure.Direction; import org.janusgraph.core.JanusGraphEdge; +import org.janusgraph.core.JanusGraphLazyEdge; import org.janusgraph.core.JanusGraphRelation; import org.janusgraph.core.JanusGraphVertex; import org.janusgraph.core.JanusGraphVertexProperty; @@ -40,13 +41,18 @@ public DirectionCondition(JanusGraphVertex vertex, Direction dir) { @Override public boolean evaluate(E element) { - if (direction==Direction.BOTH) return true; + if (direction == Direction.BOTH) return true; + + if (element instanceof JanusGraphLazyEdge) { + element = (E) ((JanusGraphLazyEdge) element).loadValue(); + } + if (element instanceof CacheEdge) { - return direction==((CacheEdge)element).getVertexCentricDirection(); + return direction == ((CacheEdge) element).getVertexCentricDirection(); } else if (element instanceof JanusGraphEdge) { - return ((JanusGraphEdge)element).vertex(direction).equals(baseVertex); + return ((JanusGraphEdge) element).vertex(direction).equals(baseVertex); } else if (element instanceof JanusGraphVertexProperty) { - return direction==Direction.OUT; + return direction == Direction.OUT; } return false; } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java index 30dc680cb87..593449a8362 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/BasicVertexCentricQueryBuilder.java @@ -27,6 +27,7 @@ import org.janusgraph.core.VertexList; import org.janusgraph.core.attribute.Cmp; import org.janusgraph.core.schema.SchemaStatus; +import org.janusgraph.diskstorage.EntryList; import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; import org.janusgraph.graphdb.database.EdgeSerializer; import org.janusgraph.graphdb.internal.ElementLifeCycle; @@ -49,10 +50,12 @@ import org.janusgraph.graphdb.query.condition.RelationTypeCondition; import org.janusgraph.graphdb.query.condition.VisibilityFilterCondition; import org.janusgraph.graphdb.query.profile.QueryProfiler; +import org.janusgraph.graphdb.relations.RelationComparator; import org.janusgraph.graphdb.relations.StandardVertexProperty; import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; import org.janusgraph.graphdb.types.system.ImplicitKey; import org.janusgraph.graphdb.types.system.SystemRelationType; +import org.janusgraph.graphdb.vertices.CacheVertex; import org.janusgraph.util.datastructures.Interval; import org.janusgraph.util.datastructures.PointInterval; import org.janusgraph.util.datastructures.RangeInterval; @@ -259,6 +262,20 @@ public Iterable emptyResult() { } + protected class CachedRelationConstructor implements ResultConstructor> { + + @Override + public Iterable getResult(InternalVertex v, BaseVertexCentricQuery bq) { + return loadRelationsFromCache(v,bq); + } + + @Override + public Iterable emptyResult() { + return Collections.emptyList(); + } + + } + protected class VertexConstructor implements ResultConstructor> { @Override @@ -326,6 +343,46 @@ protected Iterable executeRelations(InternalVertex vertex, B return executeIndividualRelations(vertex,baseQuery); } + /** + * Takes advantage of query result being cached in CachedVertex. + * Deserializes properties only on access. + * @param vertex + * @param baseQuery + * @return + */ + protected Iterable loadRelationsFromCache(InternalVertex vertex, BaseVertexCentricQuery baseQuery) { + Iterable merge = null; + if (vertex instanceof CacheVertex && !isPartitionedVertex(vertex) && vertex.isLoaded()) { + CacheVertex cacheVertex = (CacheVertex) vertex; + boolean hasNotCachedQueries = baseQuery.getQueries().stream() + .anyMatch(q -> cacheVertex.getFromCache(q.getBackendQuery()) == null || !q.isFitted()); + if (hasNotCachedQueries) { + //Fall back to default path + return executeRelations(vertex, baseQuery); + } else { + //Note: we optimize fetching relation from Vertex, if the query is already cached to avoid constructing a new query + Comparator relationComparator = new RelationComparator(vertex, baseQuery.getOrders()); + for (BackendQueryHolder bqSubQuery : baseQuery.getQueries()) { + EntryList entryList = cacheVertex.getFromCache(bqSubQuery.getBackendQuery()); + Preconditions.checkArgument(entryList != null, "Missing EntryList for Lazy load"); + Iterable iterable = org.janusgraph.graphdb.transaction.RelationConstructor.readRelation(vertex, + entryList, tx); + + if (merge == null) { + merge = iterable; + } else { + merge = ResultMergeSortIterator.mergeSort(merge, iterable, relationComparator, false); + } + } + + return ResultSetIterator.wrap(merge, baseQuery.getLimit()); + } + } else { + //Fall back to default path + return executeRelations(vertex, baseQuery); + } + } + private Iterable executeIndividualRelations(InternalVertex vertex, BaseVertexCentricQuery baseQuery) { VertexCentricQuery query = constructQuery(vertex, baseQuery); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java index 379edc158c4..f7ecd9fbe8a 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java @@ -169,14 +169,14 @@ public Map> executeImpl @Override public Map> edges() { - return (Map) execute(RelationCategory.EDGE, new RelationConstructor()); + return (Map) execute(RelationCategory.EDGE, new CachedRelationConstructor()); } @Override public Map> properties() { return (Map)(isImplicitKeyQuery(RelationCategory.PROPERTY)? executeImplicitKeyQuery(): - execute(RelationCategory.PROPERTY, new RelationConstructor())); + execute(RelationCategory.PROPERTY, new CachedRelationConstructor())); } @Override @@ -189,7 +189,7 @@ public void preFetch() { public Map> relations() { return (Map)(isImplicitKeyQuery(RelationCategory.RELATION)? executeImplicitKeyQuery(): - execute(RelationCategory.RELATION, new RelationConstructor())); + execute(RelationCategory.RELATION, new CachedRelationConstructor())); } @Override diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/RelationConstructor.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/RelationConstructor.java index d2c25be2b22..43cd113d80f 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/RelationConstructor.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/RelationConstructor.java @@ -17,6 +17,7 @@ import com.google.common.base.Preconditions; import org.apache.tinkerpop.gremlin.structure.Direction; import org.janusgraph.core.EdgeLabel; +import org.janusgraph.core.JanusGraphLazyRelationConstructor; import org.janusgraph.core.JanusGraphRelation; import org.janusgraph.core.PropertyKey; import org.janusgraph.diskstorage.Entry; @@ -41,7 +42,17 @@ public static RelationCache readRelationCache(Entry data, StandardJanusGraphTx t return tx.getEdgeSerializer().readRelation(data, false, tx); } - public static Iterable readRelation(final InternalVertex vertex, final Iterable data, final StandardJanusGraphTx tx) { + public static InternalRelation readRelation(final InternalVertex vertex, + final Entry data, + final boolean parseHeaderOnly, + final StandardJanusGraphTx tx) { + RelationCache relation = tx.getEdgeSerializer().readRelation(data, parseHeaderOnly, tx); + return readRelation(vertex,relation,data,tx,tx); + } + + public static Iterable readRelation(final InternalVertex vertex, + final Iterable data, + final StandardJanusGraphTx tx) { return () -> new Iterator() { private final Iterator iterator = data.iterator(); @@ -54,7 +65,12 @@ public boolean hasNext() { @Override public JanusGraphRelation next() { - current = readRelation(vertex, iterator.next(),tx); + Entry entry = iterator.next(); + if (tx.getConfiguration().isLazyLoadRelations()) { + current = JanusGraphLazyRelationConstructor.create(entry, vertex, tx); + } else { + current = readRelation(vertex, entry, tx); + } return current; } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java index c344795cdb0..bc0cc32f0d6 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java @@ -209,6 +209,12 @@ public class StandardJanusGraphTx extends JanusGraphBlueprintsTransaction implem */ private VertexCache vertexCache; + /** + * Keeps track of relation types vertex. Used only in {@link org.janusgraph.core.JanusGraphLazyRelation}. + * Cache is thread-safe. + */ + private final Map relTypeCache; + //######## Data structures that keep track of new and deleted elements //These data structures cannot release elements, since we would loose track of what was added or deleted /** @@ -320,6 +326,12 @@ public void close() { config.getVertexCacheSize(), effectiveVertexCacheSize, MIN_VERTEX_CACHE_SIZE); } + if(this.config.isLazyLoadRelations()) { + relTypeCache = new ConcurrentHashMap<>(30); + } else { + relTypeCache = null; + } + vertexCache = new CaffeineVertexCache(effectiveVertexCacheSize,config.getDirtyVertexSize()); indexCache = new CaffeineSubqueryCache(config.getIndexCacheWeight()); @@ -439,6 +451,16 @@ public InternalVertex[] getAllRepresentatives(JanusGraphVertex partitionedVertex return vertices; } + /* + * ------------------------------------ Relation Type Handling ------------------------------------ + */ + + public InternalRelationType getOrLoadRelationTypeById(long typeId) { + assert relTypeCache != null; + return this.relTypeCache.computeIfAbsent(typeId, (k) -> + TypeUtil.getBaseType((InternalRelationType) this.getExistingRelationType(k)) + ); + } /* * ------------------------------------ Vertex Handling ------------------------------------ diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java index 88fdfb3813d..18285ee8d5a 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java @@ -88,6 +88,8 @@ public class StandardTransactionBuilder implements TransactionConfiguration, Tra private boolean skipDBCacheRead; + private boolean isLazyLoadRelations; + private MultiQueryHasStepStrategyMode hasStepStrategyMode; private MultiQueryPropertiesStrategyMode propertiesStrategyMode; @@ -234,6 +236,12 @@ public TransactionBuilder skipDBCacheRead() { return this; } + @Override + public TransactionBuilder lazyLoadRelations() { + this.isLazyLoadRelations = true; + return this; + } + @Override public TransactionBuilder setHasStepStrategyMode(MultiQueryHasStepStrategyMode hasStepStrategyMode) { this.hasStepStrategyMode = hasStepStrategyMode; @@ -298,7 +306,7 @@ public JanusGraphTransaction start() { propertyPrefetching, multiQuery, singleThreaded, threadBound, getTimestampProvider(), userCommitTime, indexCacheWeight, getVertexCacheSize(), getDirtyVertexSize(), logIdentifier, restrictedPartitions, groupName, - defaultSchemaMaker, hasDisabledSchemaConstraints, skipDBCacheRead, hasStepStrategyMode, propertiesStrategyMode, + defaultSchemaMaker, hasDisabledSchemaConstraints, skipDBCacheRead, isLazyLoadRelations, hasStepStrategyMode, propertiesStrategyMode, labelStepStrategyMode, customOptions); return graph.newTransaction(immutable); } @@ -416,6 +424,11 @@ public boolean isSkipDBCacheRead() { return skipDBCacheRead; } + @Override + public boolean isLazyLoadRelations() { + return isLazyLoadRelations; + } + @Override public MultiQueryHasStepStrategyMode getHasStepStrategyMode() { return hasStepStrategyMode; @@ -486,6 +499,8 @@ private static class ImmutableTxCfg implements TransactionConfiguration { private final int dirtyVertexSize; private final boolean skipDBCacheRead; + + private final boolean isLazyLoadRelations; private final String logIdentifier; private final int[] restrictedPartitions; private final DefaultSchemaMaker defaultSchemaMaker; @@ -512,6 +527,7 @@ public ImmutableTxCfg(boolean isReadOnly, DefaultSchemaMaker defaultSchemaMaker, boolean hasDisabledSchemaConstraints, boolean skipDBCacheRead, + boolean isLazyLoadRelations, MultiQueryHasStepStrategyMode hasStepStrategyMode, MultiQueryPropertiesStrategyMode propertiesStrategyMode, MultiQueryLabelStepStrategyMode labelStepStrategyMode, @@ -537,6 +553,7 @@ public ImmutableTxCfg(boolean isReadOnly, this.defaultSchemaMaker = defaultSchemaMaker; this.hasDisabledSchemaConstraints = hasDisabledSchemaConstraints; this.skipDBCacheRead = skipDBCacheRead; + this.isLazyLoadRelations = isLazyLoadRelations; this.hasStepStrategyMode = hasStepStrategyMode; this.propertiesStrategyMode = propertiesStrategyMode; this.labelStepStrategyMode = labelStepStrategyMode; @@ -657,6 +674,11 @@ public boolean isSkipDBCacheRead() { return skipDBCacheRead; } + @Override + public boolean isLazyLoadRelations() { + return isLazyLoadRelations; + } + @Override public MultiQueryHasStepStrategyMode getHasStepStrategyMode() { return hasStepStrategyMode; diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java index 5959b974ab5..9f2aed31229 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java @@ -199,6 +199,12 @@ public interface TransactionConfiguration extends BaseTransactionConfig { */ boolean isSkipDBCacheRead(); + /** + * Returns true if all properties and edges are lazy-loaded: ids and values are deserialized upon demand. + * When enabled, it can have a boost on large scale read operations, when only certain type of relations are being read. + */ + boolean isLazyLoadRelations(); + /** * @return Has step strategy mode used for the transaction. Can be configured via config `query.batch.has-step-mode`. */ diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/CacheVertex.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/CacheVertex.java index 9492aea821c..46013396f50 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/CacheVertex.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/CacheVertex.java @@ -44,6 +44,10 @@ public void refresh() { } } + public EntryList getFromCache(final SliceQuery query) { + return queryCache.get(query); + } + protected void addToQueryCache(final SliceQuery query, final EntryList entries) { synchronized (queryCache) { //TODO: become smarter about what to cache and when (e.g. memory pressure) diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/PreloadedVertex.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/PreloadedVertex.java index 251036789c0..69482147f8f 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/PreloadedVertex.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/vertices/PreloadedVertex.java @@ -66,10 +66,6 @@ public void addToQueryCache(final SliceQuery query, final EntryList entries) { super.addToQueryCache(query, entries); } - public EntryList getFromCache(final SliceQuery query) { - return queryCache.get(query); - } - @Override public Iterable getAddedRelations(Predicate query) { return Collections.EMPTY_LIST; diff --git a/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java new file mode 100644 index 00000000000..b91b2482baa --- /dev/null +++ b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java @@ -0,0 +1,53 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.inmemory; + +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.janusgraph.core.JanusGraphVertex; +import org.janusgraph.diskstorage.configuration.WriteConfiguration; +import org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder; +import org.janusgraph.graphdb.database.LazyLoadGraphTest; +import org.junit.jupiter.api.Test; + +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author Matthias Broecheler (me@matthiasb.com) + */ +public class InMemoryLazyLoadGraphTest extends InMemoryGraphTest { + + @Override + public void open(WriteConfiguration config) { + graph = new LazyLoadGraphTest(new GraphDatabaseConfigurationBuilder().build(config)); + features = graph.getConfiguration().getStoreFeatures(); + tx = graph.buildTransaction().start(); + mgmt = graph.openManagement(); + } + + @Override @Test + public void testPropertyIdAccessInDifferentTransaction() { + JanusGraphVertex v1 = graph.addVertex(); + Object expectedId = v1.property("name", "foo").id(); + graph.tx().commit(); + + VertexProperty p = getOnlyElement(v1.properties("name")); + + // access property id in new transaction + graph.tx().commit(); + Exception exception = assertThrows(IllegalStateException.class, p::id); + assertEquals(exception.getMessage(), "Any lazy load operation is not supported when transaction is already closed."); + } +}