diff --git a/CHANGELOG.md b/CHANGELOG.md index 8642bc3d5..e3b4b896a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,18 @@ ##2.1.6 +* Added tests for inserting via fdw on postgres for user identified elements +* Added validation for inserting data via fdw on postgres. +* Fixed [#451](https://github.com/pietermartin/sqlg/issues/451). Postgresql copy command did not take special characters into account. +* Fixed [#450](https://github.com/pietermartin/sqlg/issues/450). Fixed a bunch of copy past errors in Sqlg's ddl statements. +* Fixed [#446](https://github.com/pietermartin/sqlg/issues/446). Upgraded H2 to 2.1.210 +* Implemented [#334](https://github.com/pietermartin/sqlg/issues/332). Added `Topology.lock` and `Topology.unlock`. +* Upgrade postgresql jdbc driver to 42.3.1 +* Implemented [#361](https://github.com/pietermartin/sqlg/issues/361). Implements UUID support on postgresql, hsqldb and h2. +* Implemented [#431](https://github.com/pietermartin/sqlg/issues/431). Optimizes `id()`. +* Implemented [#139](https://github.com/pietermartin/sqlg/issues/139). +Added `rename` to `VertexLabel`, `EdgeLabel` and `PropertyColumn` * BREAKING CHANGE: Changed `TopologyListener`'s interface, `oldValue` is now a `TopologyInf` -*Implemented [#428](https://github.com/pietermartin/sqlg/issues/428). +* Implemented [#428](https://github.com/pietermartin/sqlg/issues/428). Added ability to import foreign schemas and to import tables into an existing schema. Both import must be consistent, i.e. can not reference elements that are not being imported. diff --git a/README.md b/README.md index 8f02faffd..2e651c319 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Sqlg Currently [H2](http://h2database.com/), [HSQLDB](http://hsqldb.org/), [Postgresql](http://www.postgresql.org/), [MariaDB](https://mariadb.org/), [MySql](https://www.mysql.com/) and [MSSqlServer](https://www.microsoft.com/en-us/sql-server/sql-server-2017) are supported. +2.1.6 [Documentation](http://sqlg.org/docs/2.1.6) + 2.1.5 [Documentation](http://sqlg.org/docs/2.1.5) 2.1.4 [Documentation](http://sqlg.org/docs/2.1.4) diff --git a/pom.xml b/pom.xml index 12f3cd895..f364a2645 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.umlg sqlg pom - 2.1.6-SNAPSHOT + 2.1.7-SNAPSHOT sqlg Sqlg is a tinkerpop 3 implementation on top of a rdbms. @@ -31,10 +31,10 @@ git@github.com:pietermartin/sqlg.git - 2.1.6-SNAPSHOT + 2.1.7-SNAPSHOT 3.5.1 - 1.7.25 - 4.13.1 + 2.17.0 + 4.13.2 @@ -101,23 +101,24 @@ -Dbuild.dir=${project.build.directory} **/H2AllTest.java - - + **/SqlgH2StructureStandardTest.java + **/SqlgH2ProcessStandardTest.java + + **/SqlgHsqldbStructureStandardTest.java + **/SqlgHsqldbProcessStandardTest.java + **/HsqldbAllTest.java **/SqlgPostgresStructureStandardTest.java **/SqlgPostgresProcessStandardTest.java **/PostgresAllTest.java - **/SqlgHsqldbStructureStandardTest.java - **/SqlgHsqldbProcessStandardTest.java - **/HsqldbAllTest.java + **/SqlgMariaDBProcessStandardTest.java + **/MariadbAllTest.java - - **/MariadbAllTest.java @@ -203,6 +204,7 @@ sqlg-doc sqlg-ui sqlg-gremlin-server + sqlg-jdbc-pool @@ -248,6 +250,16 @@ sqlg-core ${sqlg.version} + + org.umlg + sqlg-c3p0 + ${sqlg.version} + + + org.umlg + sqlg-hikari + ${sqlg.version} + org.umlg sqlg-postgres @@ -341,6 +353,11 @@ c3p0 0.9.5.5 + + com.zaxxer + HikariCP + 5.0.0 + com.google.guava guava @@ -349,7 +366,7 @@ com.fasterxml.jackson.core jackson-databind - 2.10.0.pr1 + 2.13.2.1 org.apache.commons @@ -369,7 +386,7 @@ org.apache.commons commons-text - 1.9 + 1.10.0 io.projectreactor diff --git a/sqlg-core/pom.xml b/sqlg-core/pom.xml index 54c96082e..687b481a9 100755 --- a/sqlg-core/pom.xml +++ b/sqlg-core/pom.xml @@ -5,7 +5,7 @@ org.umlg sqlg - 2.1.6-SNAPSHOT + 2.1.7-SNAPSHOT sqlg-core sqlg :: core @@ -29,10 +29,6 @@ - - com.mchange - c3p0 - com.google.guava guava diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/BaseSqlDialect.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/BaseSqlDialect.java index 4c9323209..b1b856abc 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/BaseSqlDialect.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/BaseSqlDialect.java @@ -60,7 +60,7 @@ public List getSchemaNames(DatabaseMetaData metaData) { @Override public List> getVertexTables(DatabaseMetaData metaData) { List> vertexTables = new ArrayList<>(); - String[] types = new String[]{"TABLE"}; + String[] types = new String[]{"TABLE", "PARTITIONED TABLE"}; try { //load the vertices try (ResultSet vertexRs = metaData.getTables(null, null, Topology.VERTEX_PREFIX + "%", types)) { @@ -85,7 +85,7 @@ public List> getVertexTables(DatabaseMetaData met @Override public List> getEdgeTables(DatabaseMetaData metaData) { List> edgeTables = new ArrayList<>(); - String[] types = new String[]{"TABLE"}; + String[] types = new String[]{"TABLE", "PARTITIONED TABLE"}; try { //load the edges without their properties try (ResultSet edgeRs = metaData.getTables(null, null, Topology.EDGE_PREFIX + "%", types)) { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/SqlDialect.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/SqlDialect.java index a2c542b92..0d8cfdbed 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/SqlDialect.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/dialect/SqlDialect.java @@ -157,6 +157,10 @@ default boolean supportsJsonArrayValues() { return false; } + default boolean supportsUUID() { + return true; + } + default boolean supportsDurationArrayValues() { return true; } @@ -1383,4 +1387,17 @@ default String renameColumn(String schema, String table, String column, String n } return sql.toString(); } + + default String renameTable(String schema, String table, String newName) { + StringBuilder sql = new StringBuilder("ALTER TABLE "); + sql.append(maybeWrapInQoutes(schema)); + sql.append("."); + sql.append(maybeWrapInQoutes(table)); + sql.append(" RENAME TO "); + sql.append(maybeWrapInQoutes(newName)); + if (needsSemicolon()) { + sql.append(";"); + } + return sql.toString(); + } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/AndOrHasContainer.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/AndOrHasContainer.java index 24752dff3..171bc0833 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/AndOrHasContainer.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/AndOrHasContainer.java @@ -10,7 +10,6 @@ import org.umlg.sqlg.predicate.Existence; import org.umlg.sqlg.structure.PropertyType; import org.umlg.sqlg.structure.SqlgGraph; -import org.umlg.sqlg.util.SqlgUtil; import java.util.ArrayList; import java.util.List; @@ -95,42 +94,42 @@ private void toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, StringB if (!this.hasContainers.isEmpty()) { boolean first = true; for (HasContainer h : this.hasContainers) { - if (!SqlgUtil.isBulkWithin(sqlgGraph, h)) { +// if (!SqlgUtil.isBulkWithin(sqlgGraph, h)) { if (first) { first = false; result.append("("); } else { - result.append(" AND "); + result.append(" AND "); } - String k=h.getKey(); + String k = h.getKey(); WhereClause whereClause = WhereClause.from(h.getPredicate()); - + // check if property exists - String bool=null; - if (!k.equals(T.id.getAccessor())){ - Map pts=sqlgGraph.getTopology().getTableFor(schemaTableTree.getSchemaTable()); - if (pts!=null && !pts.containsKey(k)){ - // verify if we have a value - Multimap keyValueMap=LinkedListMultimap.create(); - whereClause.putKeyValueMap(h, keyValueMap, schemaTableTree); - // we do - if (keyValueMap.size()>0){ - bool="? is null"; - } else { - if (Existence.NULL.equals(h.getBiPredicate())) { - bool="1=1"; - } else { - bool="1=0"; - } - } + String bool = null; + if (!k.equals(T.id.getAccessor())) { + Map pts = sqlgGraph.getTopology().getTableFor(schemaTableTree.getSchemaTable()); + if (pts != null && !pts.containsKey(k)) { + // verify if we have a value + Multimap keyValueMap = LinkedListMultimap.create(); + whereClause.putKeyValueMap(h, keyValueMap, schemaTableTree); + // we do + if (keyValueMap.size() > 0) { + bool = "? is null"; + } else { + if (Existence.NULL.equals(h.getBiPredicate())) { + bool = "1=1"; + } else { + bool = "1=0"; + } + } } } - if (bool!=null){ - result.append(bool); + if (bool != null) { + result.append(bool); } else { - result.append(whereClause.toSql(sqlgGraph, schemaTableTree, h)); + result.append(whereClause.toSql(sqlgGraph, schemaTableTree, h, true)); } - } +// } } if (!first) { result.append(")"); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/GremlinParser.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/GremlinParser.java index 82a1244fb..145f643b2 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/GremlinParser.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/GremlinParser.java @@ -19,7 +19,7 @@ public GremlinParser(SqlgGraph sqlgGraph) { this.sqlgGraph = sqlgGraph; } - public Set parse(ReplacedStepTree replacedStepTree) { + public Set parse(ReplacedStepTree replacedStepTree) { ReplacedStep startReplacedStep = replacedStepTree.root().getReplacedStep(); Preconditions.checkState(startReplacedStep.isGraphStep(), "Step must be a GraphStep"); Set rootSchemaTableTrees = startReplacedStep.getRootSchemaTableTrees(this.sqlgGraph, replacedStepTree.getDepth()); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/ReplacedStep.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/ReplacedStep.java index da2f41961..73f595bf7 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/ReplacedStep.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/ReplacedStep.java @@ -79,6 +79,9 @@ public class ReplacedStep { */ private boolean isForSqlgSchema; + //IDStep, i.e. only return the element's id. + private boolean idOnly = false; + private ReplacedStep() { } @@ -195,7 +198,7 @@ private Set appendPathForVertexStep(SchemaTableTree schemaTable throw new IllegalStateException("Unknown direction " + direction.name()); } - Map>> groupedIds = groupIdsBySchemaTable(); + Map, RecordId>>> groupedIds = groupIdsBySchemaTable(); //Each labelToTravers more than the first one forms a new distinct path for (SchemaTable inEdgeLabelToTravers : inEdgeLabelsToTraversers) { @@ -211,7 +214,7 @@ private Set appendPathForVertexStep(SchemaTableTree schemaTable this.labels); SchemaTable schemaTable = SchemaTable.from(this.topology.getSqlgGraph(), inEdgeLabelToTravers.toString()); - List> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); addIdHasContainers(schemaTableTreeChild, biPredicateRecordIs); result.add(schemaTableTreeChild); @@ -265,7 +268,7 @@ private Set appendPathForVertexStep(SchemaTableTree schemaTable this.labels); SchemaTable schemaTable = SchemaTable.from(this.topology.getSqlgGraph(), outEdgeLabelToTravers.toString()); - List> biPredicateRecordIds = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIds = groupedIds.get(schemaTable.withOutPrefix()); addIdHasContainers(schemaTableTreeChild, biPredicateRecordIds); result.add(schemaTableTreeChild); @@ -286,7 +289,7 @@ private Set appendPathForVertexStep(SchemaTableTree schemaTable SchemaTable schemaTableTo = SchemaTable.of(foreignKeySchema, VERTEX_PREFIX + SqlgUtil.removeTrailingInId(foreignKeyTable)); if (passesLabelHasContainers(this.topology.getSqlgGraph(), true, schemaTableTo.toString()) && passesRestrictedProperties(filteredAllTables.get(schemaTableTo.toString()))) { - + if (first) { first = false; schemaTableTreeChild = schemaTableTree.addChild( @@ -308,7 +311,7 @@ private Set appendPathForVertexStep(SchemaTableTree schemaTable private Set calculatePathFromEdgeToVertex(SchemaTableTree schemaTableTree, SchemaTable labelToTravers, Direction direction) { Preconditions.checkArgument(labelToTravers.isEdgeTable()); - Map>> groupedIds = groupIdsBySchemaTable(); + Map, RecordId>>> groupedIds = groupIdsBySchemaTable(); Set result = new HashSet<>(); Map> edgeForeignKeys = this.topology.getEdgeForeignKeys(); @@ -334,7 +337,7 @@ private Set calculatePathFromEdgeToVertex(SchemaTableTree schem this, this.labels ); - List> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); addIdHasContainers(schemaTableTreeChild, biPredicateRecordIs); result.add(schemaTableTreeChild); } @@ -348,7 +351,7 @@ private Set calculatePathFromEdgeToVertex(SchemaTableTree schem this, this.labels ); - List> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); addIdHasContainers(schemaTableTreeChild, biPredicateRecordIs); result.add(schemaTableTreeChild); } @@ -357,7 +360,12 @@ private Set calculatePathFromEdgeToVertex(SchemaTableTree schem return result; } - private Set calculatePathFromVertexToEdge(SchemaTableTree schemaTableTree, SchemaTable schemaTableTo, Direction direction, Map>> groupedIds) { + private Set calculatePathFromVertexToEdge( + SchemaTableTree schemaTableTree, + SchemaTable schemaTableTo, + Direction direction, + Map, RecordId>>> groupedIds) { + Set result = new HashSet<>(); //add the child for schemaTableTo to the tree SchemaTableTree schemaTableTree1 = schemaTableTree.addChild( @@ -368,16 +376,16 @@ private Set calculatePathFromVertexToEdge(SchemaTableTree schem this.labels ); SchemaTable schemaTable = SchemaTable.from(this.topology.getSqlgGraph(), schemaTableTo.toString()); - List> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIs = groupedIds.get(schemaTable.withOutPrefix()); addIdHasContainers(schemaTableTree1, biPredicateRecordIs); result.add(schemaTableTree1); return result; } - private void addIdHasContainers(SchemaTableTree schemaTableTree1, List> biPredicateRecordIds) { + private void addIdHasContainers(SchemaTableTree schemaTableTree1, List, RecordId>> biPredicateRecordIds) { if (biPredicateRecordIds != null) { - for (Multimap biPredicateRecordId : biPredicateRecordIds) { - for (BiPredicate biPredicate : biPredicateRecordId.keySet()) { + for (Multimap, RecordId> biPredicateRecordId : biPredicateRecordIds) { + for (BiPredicate biPredicate : biPredicateRecordId.keySet()) { Collection recordIds = biPredicateRecordId.get(biPredicate); HasContainer idHasContainer; //id hasContainers are only optimized for BaseStrategy.SUPPORTED_ID_BI_PREDICATE within, without, eq, neq @@ -431,9 +439,9 @@ private Set filterVertexStepOnEdgeLabels(Set labels, S @Override public String toString() { if (this.step != null) { - return this.step.toString() + " :: " + this.hasContainers.toString() + " :: leftJoin = " + this.leftJoin + " :: joinToLeftJoin = " + this.joinToLeftJoin; + return this.step + " :: " + this.hasContainers + " :: leftJoin = " + this.leftJoin + " :: joinToLeftJoin = " + this.joinToLeftJoin; } else { - return "fakeStep :: " + this.hasContainers.toString() + " :: leftJoin = " + this.leftJoin + " :: joinToLeftJoin = " + this.joinToLeftJoin; + return "fakeStep :: " + this.hasContainers + " :: leftJoin = " + this.leftJoin + " :: joinToLeftJoin = " + this.joinToLeftJoin; } } @@ -469,7 +477,7 @@ Set getRootSchemaTableTrees(SqlgGraph sqlgGraph, int replacedSt final boolean isEdge = !isVertex; //RecordIds grouped by SchemaTable - Map>> groupedIds = null; + Map, RecordId>>> groupedIds = null; if (!this.idHasContainers.isEmpty()) { groupedIds = groupIdsBySchemaTable(); } @@ -504,7 +512,7 @@ private void collectSchemaTableTrees( SqlgGraph sqlgGraph, int replacedStepDepth, Set result, - Map>> groupedIds, + Map, RecordId>>> groupedIds, String table) { SchemaTable schemaTable = SchemaTable.from(sqlgGraph, table); @@ -512,10 +520,10 @@ private void collectSchemaTableTrees( List schemaTableTreeHasContainers = new ArrayList<>(this.hasContainers); if (groupedIds != null) { - List> biPredicateRecordIds = groupedIds.get(schemaTable.withOutPrefix()); + List, RecordId>> biPredicateRecordIds = groupedIds.get(schemaTable.withOutPrefix()); if (biPredicateRecordIds != null) { - for (Multimap biPredicateRecordId : biPredicateRecordIds) { - for (BiPredicate biPredicate : biPredicateRecordId.keySet()) { + for (Multimap, RecordId> biPredicateRecordId : biPredicateRecordIds) { + for (BiPredicate biPredicate : biPredicateRecordId.keySet()) { Collection recordIds = biPredicateRecordId.get(biPredicate); HasContainer idHasContainer; //id hasContainers are only optimized for BaseStrategy.SUPPORTED_ID_BI_PREDICATE within, without, eq, neq @@ -550,7 +558,8 @@ private void collectSchemaTableTrees( replacedStepDepth, this.labels, aggregateFunction, - groupBy + groupBy, + idOnly ); schemaTableTree.setRestrictedProperties(getRestrictedProperties()); result.add(schemaTableTree); @@ -598,7 +607,7 @@ private boolean passesLabelHasContainers(SqlgGraph sqlgGraph, boolean isVertex, } private boolean passesRestrictedProperties(Map propertyTypeMap) { - if (this.restrictedProperties == null) { + if (this.isIdOnly() || this.restrictedProperties == null) { return true; } for (String restrictedProperty : this.restrictedProperties) { @@ -616,26 +625,26 @@ private boolean passesRestrictedProperties(Map propertyTyp * * @return */ - private Map>> groupIdsBySchemaTable() { - Map>> result = new HashMap<>(); + private Map, RecordId>>> groupIdsBySchemaTable() { + Map, RecordId>>> result = new HashMap<>(); for (HasContainer idHasContainer : this.idHasContainers) { Map newHasContainerMap = new HashMap<>(); @SuppressWarnings("unchecked") P idPredicate = (P) idHasContainer.getPredicate(); - BiPredicate biPredicate = idHasContainer.getBiPredicate(); + BiPredicate biPredicate = idHasContainer.getBiPredicate(); //This is statement is for g.V().hasId(Collection) where the logic is actually P.within not P.eq if (biPredicate == Compare.eq && idPredicate.getValue() instanceof Collection && ((Collection) idPredicate.getValue()).size() > 1) { biPredicate = Contains.within; } - Multimap biPredicateRecordIdMultimap; + Multimap, RecordId> biPredicateRecordIdMultimap; if (idPredicate.getValue() instanceof Collection) { @SuppressWarnings("unchecked") Collection ids = (Collection) idPredicate.getValue(); for (Object id : ids) { RecordId recordId = RecordId.from(id); - List> biPredicateRecordIdList = result.get(recordId.getSchemaTable()); + List, RecordId>> biPredicateRecordIdList = result.get(recordId.getSchemaTable()); Boolean newHasContainer = newHasContainerMap.get(recordId.getSchemaTable()); if (biPredicateRecordIdList == null) { biPredicateRecordIdList = new ArrayList<>(); @@ -654,7 +663,7 @@ private Map>> groupIdsBySchema } else { Object id = idPredicate.getValue(); RecordId recordId = RecordId.from(id); - List> biPredicateRecordIdList = result.computeIfAbsent(recordId.getSchemaTable(), k -> new ArrayList<>()); + List, RecordId>> biPredicateRecordIdList = result.computeIfAbsent(recordId.getSchemaTable(), k -> new ArrayList<>()); biPredicateRecordIdMultimap = LinkedListMultimap.create(); biPredicateRecordIdList.add(biPredicateRecordIdMultimap); biPredicateRecordIdMultimap.put(biPredicate, recordId); @@ -869,4 +878,11 @@ public void setGroupBy(List groupBy) { this.groupBy = groupBy; } + public boolean isIdOnly() { + return this.idOnly; + } + + public void setIdOnly(boolean idOnly) { + this.idOnly = idOnly; + } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/SchemaTableTree.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/SchemaTableTree.java index cd96a901c..518cbf13b 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/SchemaTableTree.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/SchemaTableTree.java @@ -87,6 +87,7 @@ public class SchemaTableTree { private ListOrderedSet identifiers; private String distributionColumn; private boolean localStep = false; + private boolean isIdStep = false; private boolean fakeEmit = false; /** * Indicates the DropStep. @@ -106,6 +107,9 @@ public class SchemaTableTree { private List groupBy = null; private Pair> aggregateFunction = null; + //Indicates a IdStep, only the element id must be returned. + private final boolean idOnly; + SchemaTableTree(SqlgGraph sqlgGraph, SchemaTable schemaTable, int stepDepth, int replacedStepDepth) { this.sqlgGraph = sqlgGraph; this.schemaTable = schemaTable; @@ -118,6 +122,7 @@ public class SchemaTableTree { this.filteredAllTables = sqlgGraph.getTopology().getAllTables(Topology.SQLG_SCHEMA.equals(schemaTable.getSchema())); setIdentifiersAndDistributionColumn(); this.hasIDPrimaryKey = this.identifiers.isEmpty(); + this.idOnly = false; } /** @@ -143,7 +148,8 @@ public class SchemaTableTree { int replacedStepDepth, Set labels, Pair> aggregateFunction, - List groupBy + List groupBy, + boolean idOnly ) { this.sqlgGraph = sqlgGraph; this.schemaTable = schemaTable; @@ -166,6 +172,7 @@ public class SchemaTableTree { setIdentifiersAndDistributionColumn(); this.hasIDPrimaryKey = this.identifiers.isEmpty(); initializeAliasColumnNameMaps(); + this.idOnly = idOnly; } public static void constructDistinctOptionalQueries(SchemaTableTree current, List, Set>> result) { @@ -479,7 +486,9 @@ private static boolean invalidateByHas(SchemaTableTree schemaTableTree) { } private static boolean invalidateByRestrictedProperty(SchemaTableTree schemaTableTree) { - if (schemaTableTree.getRestrictedProperties() != null) { + if (schemaTableTree.idOnly) { + return false; + } else if (schemaTableTree.getRestrictedProperties() != null) { for (String restrictedProperty : schemaTableTree.getRestrictedProperties()) { if (schemaTableTree.getFilteredAllTables().get(schemaTableTree.getSchemaTable().toString()).containsKey(restrictedProperty)) { return false; @@ -2792,6 +2801,14 @@ void setLocalStep(boolean localStep) { this.localStep = localStep; } + public boolean isIdStep() { + return isIdStep; + } + + public void setIdStep(boolean idStep) { + isIdStep = idStep; + } + public boolean isLocalBarrierStep() { return localBarrierStep; } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/WhereClause.java b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/WhereClause.java index 1e8661068..2fb626a8a 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/WhereClause.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/sql/parse/WhereClause.java @@ -40,13 +40,21 @@ public static WhereClause from(P p) { } public String toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, HasContainer hasContainer) { + return toSql(sqlgGraph, schemaTableTree, hasContainer, false); + } + + public String toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, HasContainer hasContainer, boolean isInAndOrHsContainer) { String prefix = sqlgGraph.getSqlDialect().maybeWrapInQoutes(schemaTableTree.getSchemaTable().getSchema()); prefix += "."; prefix += sqlgGraph.getSqlDialect().maybeWrapInQoutes(schemaTableTree.getSchemaTable().getTable()); - return toSql(sqlgGraph, schemaTableTree, hasContainer, prefix); + return toSql(sqlgGraph, schemaTableTree, hasContainer, prefix, isInAndOrHsContainer); } public String toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, HasContainer hasContainer, String prefix) { + return toSql(sqlgGraph, schemaTableTree, hasContainer, prefix, false); + } + + public String toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, HasContainer hasContainer, String prefix, boolean isInAndOrHsContainer) { StringBuilder result = new StringBuilder(); if (p.getValue() instanceof PropertyReference && p.getBiPredicate() instanceof Compare) { @@ -74,7 +82,9 @@ public String toSql(SqlgGraph sqlgGraph, SchemaTableTree schemaTableTree, HasCon result.append(compareToSql((Compare) p.getBiPredicate())); } return result.toString(); - } else if ((!sqlgGraph.getSqlDialect().supportsBulkWithinOut() || (!SqlgUtil.isBulkWithinAndOut(sqlgGraph, hasContainer))) && p.getBiPredicate() instanceof Contains) { + } else if ((!sqlgGraph.getSqlDialect().supportsBulkWithinOut() || (!SqlgUtil.isBulkWithinAndOut(sqlgGraph, hasContainer)) || isInAndOrHsContainer) && + p.getBiPredicate() instanceof Contains) { + if (hasContainer.getKey().equals(T.id.getAccessor())) { if (schemaTableTree.isHasIDPrimaryKey()) { result.append(prefix).append(".").append(sqlgGraph.getSqlDialect().maybeWrapInQoutes("ID")); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/step/SqlgIdStep.java b/sqlg-core/src/main/java/org/umlg/sqlg/step/SqlgIdStep.java new file mode 100644 index 000000000..5b6ce025c --- /dev/null +++ b/sqlg-core/src/main/java/org/umlg/sqlg/step/SqlgIdStep.java @@ -0,0 +1,18 @@ +package org.umlg.sqlg.step; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Element; + +public class SqlgIdStep extends SqlgMapStep implements TraversalParent { + + public SqlgIdStep(Traversal.Admin traversal) { + super(traversal); + } + + @Override + protected Object map(Traverser.Admin traverser) { + return traverser.get().id(); + } +} diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/strategy/BaseStrategy.java b/sqlg-core/src/main/java/org/umlg/sqlg/strategy/BaseStrategy.java index 4176eeb77..7d33c94b7 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/strategy/BaseStrategy.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/strategy/BaseStrategy.java @@ -79,7 +79,8 @@ public abstract class BaseStrategy { MeanGlobalStep.class, CountGlobalStep.class, GroupStep.class, - GroupCountStep.class + GroupCountStep.class, + IdStep.class // FoldStep.class ); @@ -221,6 +222,8 @@ boolean handleStep(ListIterator> stepIterator, MutableInt pathCount) return handlePropertiesStep(this.currentReplacedStep, step); } else if (step instanceof PropertyMapStep) { return handlePropertyMapStep(step); + } else if (step instanceof IdStep) { + return handleIdStep(step); } else if (step instanceof MaxGlobalStep) { return handleAggregateGlobalStep(this.currentReplacedStep, step, max); } else if (step instanceof MinGlobalStep) { @@ -259,6 +262,39 @@ private void removeTinkerPopLabels(Step step) { } } + private boolean handleIdStep(Step step) { + Step dropStep = SqlgTraversalUtil.stepAfter(this.traversal, DropStep.class, step); + if (dropStep != null) { + return false; + } + Step orderGlobalStep = SqlgTraversalUtil.stepAfter(this.traversal, OrderGlobalStep.class, step); + if (orderGlobalStep != null) { + return false; + } + Step selectOneStep = SqlgTraversalUtil.stepAfter(this.traversal, SelectOneStep.class, step); + if (selectOneStep != null) { + return false; + } + Step selectStep = SqlgTraversalUtil.stepAfter(this.traversal, SelectStep.class, step); + if (selectStep != null) { + return false; + } + Step lambdaStep = SqlgTraversalUtil.lastLambdaHolderBefore(this.traversal, step); + if (lambdaStep == null) { + SqlgIdStep sqlgIdStep = new SqlgIdStep(traversal); + for (String label : step.getLabels()) { + sqlgIdStep.addLabel(label); + } + //noinspection unchecked,rawtypes + TraversalHelper.replaceStep((Step) step, sqlgIdStep, traversal); + this.currentReplacedStep.setIdOnly(true); + if (this.currentReplacedStep.getRestrictedProperties() == null) { + this.currentReplacedStep.setRestrictedProperties(new HashSet<>()); + } + } + return true; + } + private boolean handlePropertyMapStep(Step step) { Step dropStep = SqlgTraversalUtil.stepAfter(this.traversal, DropStep.class, step); if (dropStep != null) { @@ -789,6 +825,8 @@ private Optional handleConnectiveStepInternal(ConnectiveStep boolean hasContainerKeyNotIdOrLabel = hasContainerKeyNotIdOrLabel(hasContainer); if (hasContainerKeyNotIdOrLabel && SUPPORTED_BI_PREDICATE.contains(hasContainer.getBiPredicate())) { andOrHasContainer.addHasContainer(hasContainer); + } else if (hasContainerKeyNotIdOrLabel && (hasContainer.getBiPredicate().equals(Contains.within) || hasContainer.getBiPredicate().equals(Contains.without))) { + andOrHasContainer.addHasContainer(hasContainer); } else if (hasContainerKeyNotIdOrLabel && hasContainer.getPredicate() instanceof AndP) { AndP andP = (AndP) hasContainer.getPredicate(); List> predicates = andP.getPredicates(); @@ -1337,7 +1375,7 @@ private boolean handleGroupCountStep(ReplacedStep replacedStep, Step if (localChildren.size() == 1) { List groupByKeys = new ArrayList<>(); Traversal.Admin groupByCountTraversal = localChildren.get(0); - if (groupByCountTraversal instanceof ValueTraversal) { + if (groupByCountTraversal instanceof ValueTraversal) { ValueTraversal elementValueTraversal = (ValueTraversal) groupByCountTraversal; groupByKeys.add(elementValueTraversal.getPropertyKey()); replacedStep.setRestrictedProperties(new HashSet<>(groupByKeys)); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/PropertyType.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/PropertyType.java index 5e55096a9..6145164dd 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/PropertyType.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/PropertyType.java @@ -101,6 +101,8 @@ public class PropertyType { public static final PropertyType JSON_ARRAY = new PropertyType("JSON_ARRAY", JSON_ARRAY_ORDINAL, JsonNode[].class.getName(), new String[]{}); public static final int VARCHAR_ORDINAL = 42; + public static final int UUID_ORDINAL = 43; + public static final PropertyType UUID = new PropertyType("UUID", UUID_ORDINAL, java.util.UUID.class.getName(), new String[]{}); public static PropertyType varChar(int length) { return new PropertyType(String.class.getName(), new String[]{}, length); @@ -196,6 +198,7 @@ public static PropertyType[] values() { PropertyType.POLYGON, PropertyType.GEOGRAPHY_POINT, PropertyType.GEOGRAPHY_POLYGON, + PropertyType.UUID, PropertyType.boolean_ARRAY, PropertyType.BOOLEAN_ARRAY, PropertyType.byte_ARRAY, diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/RecordId.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/RecordId.java index 42a99700c..9514a1470 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/RecordId.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/RecordId.java @@ -122,11 +122,11 @@ private static RecordId recordIdFromIdentifiers(SqlgGraph sqlgGraph, String labe AbstractLabel abstractLabel; Optional vertexLabel = sqlgGraph.getTopology().getSchema(schemaTable.getSchema()).orElseThrow(() -> new IllegalStateException(String.format("Schema %s not found.", schemaTable.getSchema()))) .getVertexLabel(schemaTable.getTable()); - if (!vertexLabel.isPresent()) { + if (vertexLabel.isEmpty()) { Optional edgeLabel = sqlgGraph.getTopology().getSchema(schemaTable.getSchema()).orElseThrow(() -> new IllegalStateException(String.format("Schema %s not found.", schemaTable.getSchema()))) .getEdgeLabel(schemaTable.getTable()); - if (!edgeLabel.isPresent()) { - throw new IllegalStateException(String.format("SchemaTable %s not found", schemaTable.toString())); + if (edgeLabel.isEmpty()) { + throw new IllegalStateException(String.format("SchemaTable %s not found", schemaTable)); } abstractLabel = edgeLabel.get(); } else { @@ -142,7 +142,7 @@ private static RecordId recordIdFromIdentifiers(SqlgGraph sqlgGraph, String labe Comparable value = (Comparable) SqlgUtil.stringValueToType(propertyType, identifiers[count++].trim()); identifierValues.add(value); } else { - throw new IllegalStateException(String.format("identifier %s for %s not found", identifier, schemaTable.toString())); + throw new IllegalStateException(String.format("identifier %s for %s not found", identifier, schemaTable)); } } return RecordId.from(schemaTable, identifierValues); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSource.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSource.java index 469b49236..1263c4fc9 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSource.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSource.java @@ -6,6 +6,8 @@ import javax.sql.DataSource; public interface SqlgDataSource { + String C3P0DataSource = "org.umlg.sqlg.structure.ds.SqlgC3P0DataSource"; + String SqlgHikariDataSource = "org.umlg.sqlg.structure.ds.SqlgHikariDataSource"; DataSource getDatasource(); SqlDialect getDialect(); @@ -47,4 +49,11 @@ static boolean isMysql(Configuration configuration) { return configuration.getString("jdbc.url").contains("mysql"); } + default boolean isC3p0() { + return false; + } + + default boolean isHikari() { + return false; + } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSourceFactory.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSourceFactory.java index 4d66634c9..16b64a729 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSourceFactory.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgDataSourceFactory.java @@ -1,8 +1,8 @@ package org.umlg.sqlg.structure; +import com.google.common.base.Preconditions; import org.apache.commons.configuration2.Configuration; import org.apache.tinkerpop.gremlin.structure.Graph; -import org.umlg.sqlg.structure.ds.C3P0DataSource; import org.umlg.sqlg.structure.ds.JNDIDataSource; import java.lang.reflect.InvocationTargetException; @@ -14,26 +14,27 @@ */ public class SqlgDataSourceFactory { public static SqlgDataSource create(final Configuration configuration) { - if (null == configuration) + if (null == configuration) { throw Graph.Exceptions.argumentCanNotBeNull("configuration"); - + } + Preconditions.checkState(!configuration.containsKey("cache.vertices"), "\"cache.vertices\" is not supported from Sqlg 2.1.6"); try { - - final String clazz = configuration.getString(SqlgGraph.DATA_SOURCE, null); + Class dataSourceClass; + String clazz = configuration.getString(SqlgGraph.DATA_SOURCE, null); if (null == clazz) { final String jdbcUrl = configuration.getString(JDBC_URL, ""); if (JNDIDataSource.isJNDIUrl(jdbcUrl)) { return JNDIDataSource.create(configuration); - } else { - return C3P0DataSource.create(configuration); } - } - - final Class dataSourceClass; - try { + clazz = SqlgDataSource.C3P0DataSource; + try { + dataSourceClass = Class.forName(clazz); + } catch (final ClassNotFoundException e) { + clazz = SqlgDataSource.SqlgHikariDataSource; + dataSourceClass = Class.forName(clazz); + } + } else { dataSourceClass = Class.forName(clazz); - } catch (final ClassNotFoundException e) { - throw new RuntimeException(String.format("SqlgDataSourceFactory could not find [%s] - Ensure that the jar is in the classpath", clazz)); } final SqlgDataSource dataSource; @@ -48,7 +49,7 @@ public static SqlgDataSource create(final Configuration configuration) { throw new RuntimeException(String.format("SqlgDataSourceFactory could not instantiate this SqlgDataSource implementation [%s]", dataSourceClass.getName()), e3); } return dataSource; - + } catch (Exception ex) { // Exception handling preserves an existing behavior throw new IllegalStateException("Could not create sqlg data source.", ex); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgElement.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgElement.java index 9079ca64c..cf5bbe676 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgElement.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgElement.java @@ -695,6 +695,14 @@ public boolean loadProperty( } else { return false; } + case UUID_ORDINAL: + UUID uuid = (UUID)resultSet.getObject(columnIndex); + if (!resultSet.wasNull()) { + this.properties.put(propertyName, uuid); + return true; + } else { + return false; + } case boolean_ARRAY_ORDINAL: java.sql.Array array = resultSet.getArray(columnIndex); if (array != null) { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgGraph.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgGraph.java index 462bbb419..210e85712 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgGraph.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgGraph.java @@ -232,6 +232,10 @@ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest", method = "g_io_writeXkryoX", reason = "Needs to register SqlgIoRegistryV3, this test is duplicated in TestIo") +@OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXk_withinXcXX_valuesXkX", + reason = "Fails for MariaDb, the test is copied to TestHas for the other dbs") public class SqlgGraph implements Graph { public static final String DATA_SOURCE = "sqlg.dataSource"; @@ -341,7 +345,7 @@ private SqlgGraph(final Configuration configuration, SqlgDataSource dataSource) } catch (Exception e) { throw new RuntimeException(e); } - this.sqlgTransaction = new SqlgTransaction(this, this.configuration.getBoolean("cache.vertices", false)); + this.sqlgTransaction = new SqlgTransaction(this); // read fetch size from configuration, use default as specified in the dialect // this can be very useful for Postgres since according to < https://jdbc.postgresql.org/documentation/head/query.html#query-with-cursor> @@ -419,6 +423,8 @@ public Vertex addVertex(Object... keyValues) { VertexLabel vertexLabel = this.getTopology().ensureVertexLabelExist(schemaTablePair.getSchema(), schemaTablePair.getTable(), columns); if (!vertexLabel.hasIDPrimaryKey()) { Preconditions.checkArgument(columns.keySet().containsAll(vertexLabel.getIdentifiers()), "identifiers must be present %s", vertexLabel.getIdentifiers()); + } else if (vertexLabel.isForeign()) { + throw SqlgExceptions.invalidMode("Foreign VertexLabel must have user defined identifiers to support addition."); } return new SqlgVertex(this, false, false, schemaTablePair.getSchema(), schemaTablePair.getTable(), keyValueMapPair); } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgStartupManager.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgStartupManager.java index 7698df837..6aa971fc9 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgStartupManager.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgStartupManager.java @@ -169,6 +169,10 @@ private void updateTopology(String oldVersion) { if (v.isUnknownVersion() || v.compareTo(new Version(2, 1, 5, null, null, null)) < 0) { removeGlobalUniqueIndexFromSqlgSchema(); } + if (v.isUnknownVersion() || v.compareTo(new Version(2, 1, 6, null, null, null)) < 0) { + //https://github.com/pietermartin/sqlg/issues/450 + correctSqlgSchemaDDLOnForeignKeyIndexes(); + } } private void addPartitionSupportToSqlgSchema() { @@ -221,6 +225,34 @@ private void removeGlobalUniqueIndexFromSqlgSchema() { } + //https://github.com/pietermartin/sqlg/issues/450 + private void correctSqlgSchemaDDLOnForeignKeyIndexes() { + List result = new ArrayList<>(); + result.add("CREATE INDEX IF NOT EXISTS \"" + Topology.EDGE_PREFIX + "_edge_identifier_property__I_idx\" ON \"sqlg_schema\".\"E_edge_identifier\" (\"sqlg_schema.property__I\");"); + result.add("ALTER INDEX IF EXISTS \"sqlg_schema\".\"" + Topology.EDGE_PREFIX + "_vertex_identifier_edge__O_idx\" RENAME TO \"" + Topology.EDGE_PREFIX + "_edge_identifier_edge__O_idx\";"); + + result.add("ALTER INDEX IF EXISTS \"sqlg_schema\".\"" + Topology.EDGE_PREFIX + "_vertex_partition_edge__O_idx\" RENAME TO \"" + Topology.EDGE_PREFIX + "_edge_partition_edge__O_idx\";"); + result.add("CREATE INDEX IF NOT EXISTS \"" + Topology.EDGE_PREFIX + "_edge_partition_partition__I_idx\" ON \"sqlg_schema\".\"E_edge_partition\" (\"sqlg_schema.partition__I\");"); + + result.add("ALTER INDEX IF EXISTS \"sqlg_schema\".\"" + Topology.EDGE_PREFIX + "_vertex_partition_partition__O_idx\" RENAME TO \"" + Topology.EDGE_PREFIX + "_partition_partition_partition__O_idx\";"); + result.add("CREATE INDEX IF NOT EXISTS \"" + Topology.EDGE_PREFIX + "_partition_partition_partition__I_idx\" ON \"sqlg_schema\".\"E_partition_partition\" (\"sqlg_schema.partition__I\");"); + + Connection conn = this.sqlgGraph.tx().getConnection(); + try (Statement statement = conn.createStatement()) { + //Hsqldb can not do this in one go + for (String script: result) { + if (logger.isDebugEnabled()) { + logger.debug(script); + } + statement.execute(script); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + + + } + private void upgradeForeignKeysToDeferrable() { Connection conn = this.sqlgGraph.tx().getConnection(); try (PreparedStatement s = conn.prepareStatement(this.sqlDialect.sqlToGetAllForeignKeys())) { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgTransaction.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgTransaction.java index 99ce20e0d..8d4df3f2a 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgTransaction.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgTransaction.java @@ -8,11 +8,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.umlg.sqlg.sql.dialect.SqlBulkDialect; +import org.umlg.sqlg.structure.topology.Topology; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; /** * This class is a singleton. Instantiated and owned by SqlGraph. @@ -31,22 +32,22 @@ public class SqlgTransaction extends AbstractThreadLocalTransaction { private AfterCommit afterCommitFunction; private AfterRollback afterRollbackFunction; private static final Logger LOGGER = LoggerFactory.getLogger(SqlgTransaction.class); - private final boolean cacheVertices; private final ThreadLocal threadLocalTx = ThreadLocal.withInitial(() -> null); private final ThreadLocal threadLocalPreparedStatementTx = ThreadLocal.withInitial(PreparedStatementCache::new); + private final ThreadLocal threadLocalTopologyLocked = ThreadLocal.withInitial(() -> new AtomicBoolean(true)); + /** * default fetch size */ private Integer defaultFetchSize = null; - SqlgTransaction(Graph sqlgGraph, boolean cacheVertices) { + SqlgTransaction(Graph sqlgGraph) { super(sqlgGraph); this.sqlgGraph = (SqlgGraph) sqlgGraph; - this.cacheVertices = cacheVertices; } @Override @@ -70,9 +71,9 @@ protected void doOpen() { boolean lazy = this.sqlgGraph.getConfiguration().getBoolean(QUERY_LAZY, true); TransactionCache tc; if (supportsBatchMode()) { - tc = TransactionCache.of(this.cacheVertices, connection, new BatchManager(this.sqlgGraph, ((SqlBulkDialect) this.sqlgGraph.getSqlDialect())), lazy); + tc = TransactionCache.of(connection, new BatchManager(this.sqlgGraph, ((SqlBulkDialect) this.sqlgGraph.getSqlDialect())), lazy); } else { - tc = TransactionCache.of(this.cacheVertices, connection, lazy); + tc = TransactionCache.of(connection, lazy); } tc.setFetchSize(getDefaultFetchSize()); this.threadLocalTx.set(tc); @@ -89,6 +90,7 @@ protected void doCommit() throws TransactionException { } Connection connection = null; try { + this.threadLocalTopologyLocked.get().set(true); if (supportsBatchMode() && this.threadLocalTx.get().getBatchManager().isInBatchMode()) { getBatchManager().flush(); } @@ -132,6 +134,7 @@ protected void doRollback() throws TransactionException { } Connection connection = null; try { + this.threadLocalTopologyLocked.get().set(true); if (supportsBatchMode() && this.threadLocalTx.get().getBatchManager().isInBatchMode()) { try { this.threadLocalTx.get().getBatchManager().close(); @@ -169,6 +172,21 @@ protected void doRollback() throws TransactionException { } } + /** + * If {@link Topology#isLocked()} then this method will unlock the {@link Topology} for the duration of the transaction. + * It will automatically be locked again on commit or rollback. + */ + public void unlockTopology() { + this.threadLocalTopologyLocked.get().set(false); + } + + /** + * @return Returns false if the topology is unlocked or the current transaction has not unlocked it. + */ + public boolean isTopologyLocked() { + return this.sqlgGraph.getTopology().isLocked() && this.threadLocalTopologyLocked.get().get(); + } + public void streamingWithLockBatchModeOn() { if (supportsBatchMode()) { readWrite(); @@ -293,24 +311,6 @@ public boolean isOpen() { return this.threadLocalTx.get() != null; } - SqlgVertex putVertexIfAbsent(SqlgGraph sqlgGraph, String schema, String table, Long id) { - return this.threadLocalTx.get().putVertexIfAbsent(sqlgGraph, schema, table, id); - } - - SqlgVertex putVertexIfAbsent(SqlgGraph sqlgGraph, String schema, String table, List identifiers) { - return this.threadLocalTx.get().putVertexIfAbsent(sqlgGraph, schema, table, identifiers); - } - - //Called for vertices that exist but are not yet in the transaction cache - SqlgVertex putVertexIfAbsent(SqlgVertex sqlgVertex) { - return this.threadLocalTx.get().putVertexIfAbsent(sqlgVertex); - } - - //Called for new vertices - void add(SqlgVertex sqlgVertex) { - this.threadLocalTx.get().add(sqlgVertex); - } - public void add(PreparedStatement preparedStatement) { this.threadLocalPreparedStatementTx.get().add(preparedStatement); } @@ -343,7 +343,7 @@ public void setLazyQueries(boolean lazy) { /** * get default fetch size * - * @return + * @return The default jdbc fetch size */ private Integer getDefaultFetchSize() { return defaultFetchSize; @@ -352,7 +352,7 @@ private Integer getDefaultFetchSize() { /** * set default fetch size * - * @param fetchSize + * @param fetchSize Set the jdbc fetch size */ public void setDefaultFetchSize(Integer fetchSize) { this.defaultFetchSize = fetchSize; @@ -361,7 +361,7 @@ public void setDefaultFetchSize(Integer fetchSize) { /** * get fetch size for current transaction * - * @return + * @return The jdbc fetch for the current transaction */ public Integer getFetchSize() { readWrite(); @@ -371,7 +371,7 @@ public Integer getFetchSize() { /** * set fetch size for current transaction * - * @param fetchSize + * @param fetchSize Set the current transaction's jdbc fetch size */ public void setFetchSize(Integer fetchSize) { readWrite(); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgVertex.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgVertex.java index 5d3d5a9f3..19a456e03 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgVertex.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/SqlgVertex.java @@ -44,9 +44,6 @@ public SqlgVertex( super(sqlgGraph, schema, table); insertVertex(temporary, streaming, keyValueMapPair); - if (!sqlgGraph.tx().isInBatchMode()) { - sqlgGraph.tx().add(this); - } } SqlgVertex(SqlgGraph sqlgGraph, String table, Map keyValueMap) { @@ -56,19 +53,11 @@ public SqlgVertex( } public static SqlgVertex of(SqlgGraph sqlgGraph, Long id, String schema, String table) { - if (!sqlgGraph.tx().isInBatchMode()) { - return sqlgGraph.tx().putVertexIfAbsent(sqlgGraph, schema, table, id); - } else { - return new SqlgVertex(sqlgGraph, id, schema, table); - } + return new SqlgVertex(sqlgGraph, id, schema, table); } public static SqlgVertex of(SqlgGraph sqlgGraph, List identifiers, String schema, String table) { - if (!sqlgGraph.tx().isInBatchMode()) { - return sqlgGraph.tx().putVertexIfAbsent(sqlgGraph, schema, table, identifiers); - } else { - return new SqlgVertex(sqlgGraph, identifiers, schema, table); - } + return new SqlgVertex(sqlgGraph, identifiers, schema, table); } /** @@ -170,6 +159,8 @@ private Edge addEdgeInternal(boolean complete, String label, Vertex inVertex, Ob EdgeLabel edgeLabel = this.sqlgGraph.getTopology().ensureEdgeLabelExist(label, outVertexLabelOptional.get(), inVertexLabelOptional.get(), columns); if (!edgeLabel.hasIDPrimaryKey()) { Preconditions.checkArgument(columns.keySet().containsAll(edgeLabel.getIdentifiers()), "identifiers must be present %s", edgeLabel.getIdentifiers()); + } else if (edgeLabel.isForeign()) { + throw SqlgExceptions.invalidMode("Foreign EdgeLabel must have user defined identifiers to support addition."); } return new SqlgEdge(this.sqlgGraph, complete, this.schema, label, (SqlgVertex) inVertex, this, keyValueMapPair); } @@ -187,13 +178,6 @@ public VertexProperty property(final String key) { if (this.removed) { throw new IllegalStateException(String.format("Vertex with id %s was removed.", id().toString())); } else { - if (!sqlgGraph.tx().isInBatchMode()) { - SqlgVertex sqlgVertex = this.sqlgGraph.tx().putVertexIfAbsent(this); - if (sqlgVertex != this) { - //sync the properties - this.properties = sqlgVertex.properties; - } - } return (VertexProperty) super.property(key); } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/TransactionCache.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/TransactionCache.java index 02ba3ee07..f2f04644d 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/TransactionCache.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/TransactionCache.java @@ -2,7 +2,6 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.List; import java.util.Map; import java.util.WeakHashMap; @@ -16,8 +15,6 @@ class TransactionCache { private final Connection connection; private final Map elementPropertyRollbackFunctions = new WeakHashMap<>(); private BatchManager batchManager; - private final boolean cacheVertices; - private final Map vertexCache = new WeakHashMap<>(); private boolean writeTransaction; /** @@ -31,31 +28,28 @@ class TransactionCache { private Integer fetchSize = null; - static TransactionCache of(boolean cacheVertices, Connection connection, BatchManager batchManager, boolean lazyQueries) { - return new TransactionCache(cacheVertices, connection, batchManager, lazyQueries); + static TransactionCache of(Connection connection, BatchManager batchManager, boolean lazyQueries) { + return new TransactionCache(connection, batchManager, lazyQueries); } - static TransactionCache of(boolean cacheVertices, Connection connection, boolean lazyQueries) { - return new TransactionCache(cacheVertices, connection, lazyQueries); + static TransactionCache of(Connection connection, boolean lazyQueries) { + return new TransactionCache(connection, lazyQueries); } private TransactionCache( - boolean cacheVertices, Connection connection, boolean lazyQueries) { - this.cacheVertices = cacheVertices; this.connection = connection; this.lazyQueries = lazyQueries; } private TransactionCache( - boolean cacheVertices, Connection connection, BatchManager batchManager, boolean lazyQueries) { - this(cacheVertices, connection, lazyQueries); + this(connection, lazyQueries); this.batchManager = batchManager; } @@ -84,9 +78,6 @@ void clear() { if (this.batchManager != null) { this.batchManager.clear(); } - if (this.cacheVertices) { - this.vertexCache.clear(); - } try { if (!this.connection.isClosed()) { this.connection.close(); @@ -97,88 +88,7 @@ void clear() { } /** - * The recordId is not referenced in the SqlgVertex. - * It is important that the value of the WeakHashMap does not reference the key. - * - * @param sqlgGraph The graph - * @return the vertex. If cacheVertices is true and the vertex is cached then the cached vertex will be returned else - * a the vertex will be instantiated. - */ - SqlgVertex putVertexIfAbsent(SqlgGraph sqlgGraph, String schema, String table, Long id) { - RecordId recordId = RecordId.from(SchemaTable.of(schema, table), id); - SqlgVertex sqlgVertex; - if (this.cacheVertices) { - sqlgVertex = this.vertexCache.get(recordId); - if (sqlgVertex == null) { - sqlgVertex = new SqlgVertex(sqlgGraph, id, schema, table); - this.vertexCache.put(recordId, sqlgVertex); - return sqlgVertex; - } - } else { - sqlgVertex = new SqlgVertex(sqlgGraph, id, schema, table); - } - return sqlgVertex; - } - - SqlgVertex putVertexIfAbsent(SqlgGraph sqlgGraph, String schema, String table, List identifiers) { - RecordId recordId = RecordId.from(SchemaTable.of(schema, table), identifiers); - SqlgVertex sqlgVertex; - if (this.cacheVertices) { - sqlgVertex = this.vertexCache.get(recordId); - if (sqlgVertex == null) { - sqlgVertex = new SqlgVertex(sqlgGraph, identifiers, schema, table); - this.vertexCache.put(recordId, sqlgVertex); - return sqlgVertex; - } - } else { - sqlgVertex = new SqlgVertex(sqlgGraph, identifiers, schema, table); - } - return sqlgVertex; - } - - SqlgVertex putVertexIfAbsent(SqlgVertex sqlgVertex) { - RecordId vertexRecordId = (RecordId) sqlgVertex.id(); - SqlgVertex sqlgVertexFromCache; - if (this.cacheVertices) { - sqlgVertexFromCache = this.vertexCache.get(vertexRecordId); - if (sqlgVertexFromCache == null) { - //copy the RecordId so that the WeakHashMap value does not reference the key - SchemaTable schemaTable = vertexRecordId.getSchemaTable(); - RecordId recordId; - if (vertexRecordId.hasSequenceId()) { - recordId = RecordId.from(SchemaTable.of(schemaTable.getSchema(), schemaTable.getTable()), vertexRecordId.sequenceId()); - } else { - recordId = RecordId.from(SchemaTable.of(schemaTable.getSchema(), schemaTable.getTable()), vertexRecordId.getIdentifiers()); - } - this.vertexCache.put(recordId, sqlgVertex); - return sqlgVertex; - } else { - return sqlgVertexFromCache; - } - - } else { - return sqlgVertex; - } - } - - void add(SqlgVertex sqlgVertex) { - RecordId vertexRecordId = (RecordId) sqlgVertex.id(); - if (this.vertexCache.containsKey(vertexRecordId)) { - throw new IllegalStateException("The vertex cache should never already contain a new vertex!"); - } else { - SchemaTable schemaTable = vertexRecordId.getSchemaTable(); - RecordId recordId; - if (vertexRecordId.hasSequenceId()) { - recordId = RecordId.from(SchemaTable.of(schemaTable.getSchema(), schemaTable.getTable()), vertexRecordId.sequenceId()); - } else { - recordId = RecordId.from(SchemaTable.of(schemaTable.getSchema(), schemaTable.getTable()), vertexRecordId.getIdentifiers()); - } - this.vertexCache.put(recordId, sqlgVertex); - } - } - - /** - * are we reading the SQL query results laszily? + * are we reading the SQL query results lazily? * * @return true if we are processing the results lazily, false otherwise */ @@ -189,7 +99,7 @@ public boolean isLazyQueries() { /** * set the laziness on query result reading * - * @param lazyQueries + * @param lazyQueries set the queries to being lazy. */ public void setLazyQueries(boolean lazyQueries) { this.lazyQueries = lazyQueries; diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/ds/JNDIDataSource.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/ds/JNDIDataSource.java index bd59691fa..4acb381ce 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/ds/JNDIDataSource.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/ds/JNDIDataSource.java @@ -29,8 +29,8 @@ public static boolean isJNDIUrl(String url) { public static SqlgDataSource create(Configuration configuration) throws NamingException, SQLException { String url = configuration.getString(SqlgGraph.JDBC_URL); - if (! isJNDIUrl(url)) { - throw new IllegalArgumentException("Creating JNDI ds from invalid url: "+url); + if (!isJNDIUrl(url)) { + throw new IllegalArgumentException("Creating JNDI ds from invalid url: " + url); } String jndiName = url.substring(JNDI_PREFIX.length()); @@ -69,18 +69,14 @@ public void close() { @Override public String getPoolStatsAsJson() { try { - StringBuilder json = new StringBuilder(); - json.append("["); - - json.append("{\"jdbcUrl\":\"").append(jdbcUrl).append("\","); - json.append("\"jndi\": true"); - json.append("}"); - - - json.append("]"); - return json.toString(); + return "[" + + "{\"jdbcUrl\":\"" + jdbcUrl + "\"," + + "\"jndi\": true" + + "}" + + "]"; } catch (Exception e) { throw new IllegalStateException("Json generation failed", e); } } + } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/AbstractLabel.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/AbstractLabel.java index 336fdef82..1cce2696f 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/AbstractLabel.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/AbstractLabel.java @@ -39,7 +39,7 @@ public abstract class AbstractLabel implements TopologyInf { final Set uncommittedRemovedProperties = new ThreadLocalSet<>(); private final TreeMap identifierMap = new TreeMap<>(); - private final ListOrderedSet identifiers = new ListOrderedSet<>(); + final ListOrderedSet identifiers = new ListOrderedSet<>(); private final Set uncommittedIdentifiers = new ThreadLocalListOrderedSet<>(); //Pair of identifiers final Set> renamedIdentifiers = new ThreadLocalListOrderedSet<>(); @@ -63,7 +63,7 @@ public abstract class AbstractLabel implements TopologyInf { */ PartitionType partitionType = PartitionType.NONE; String partitionExpression; - private final Map partitions = new HashMap<>(); + private final Map partitions = new ConcurrentHashMap<>(); private final Map uncommittedPartitions = new ThreadLocalMap<>(); private final Set uncommittedRemovedPartitions = new ThreadLocalSet<>(); @@ -157,9 +157,10 @@ public Partition ensureRangePartitionExists(String name, String from, String to) Objects.requireNonNull(from, "RANGE Partition's \"from\" must not be null"); Objects.requireNonNull(to, "RANGE Partition's \"to\" must not be null"); Preconditions.checkState(this.partitionType == PartitionType.RANGE, "ensureRangePartitionExists(String name, String from, String to) can only be called for a RANGE partitioned VertexLabel. Found %s", this.partitionType.name()); + this.sqlgGraph.getSqlDialect().validateTableName(name); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createRangePartition(name, from, to)); } else { @@ -185,9 +186,10 @@ public Partition ensureRangePartitionWithSubPartitionExists(String name, String Objects.requireNonNull(partitionType, "Sub-partition's \"partitionType\" must not be null"); Objects.requireNonNull(partitionExpression, "Sub-partition's \"partitionExpression\" must not be null"); Preconditions.checkState(this.partitionType == PartitionType.RANGE, "ensureRangePartitionExists(String name, String from, String to, PartitionType partitionType, String partitionExpression) can only be called for a RANGE partitioned VertexLabel. Found %s", this.partitionType.name()); + this.sqlgGraph.getSqlDialect().validateTableName(name); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createRangePartitionWithSubPartition(name, from, to, partitionType, partitionExpression)); } else { @@ -203,6 +205,7 @@ public Partition ensureRangePartitionWithSubPartitionExists(String name, String * @param remainder The HASH partition's 'remainder'. * @return The {@link Partition} */ + @SuppressWarnings("UnusedReturnValue") public Partition ensureHashPartitionExists(String name, int modulus, int remainder) { Preconditions.checkState(this.sqlgGraph.getSqlDialect().supportsPartitioning()); Objects.requireNonNull(name, "HASH Partition's \"name\" must not be null"); @@ -211,7 +214,7 @@ public Partition ensureHashPartitionExists(String name, int modulus, int remaind Preconditions.checkState(this.partitionType == PartitionType.HASH, "ensureHashPartitionExists(String name, String ... in) can only be called for a LIST partitioned VertexLabel. Found %s", this.partitionType.name()); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createHashPartition(name, modulus, remainder)); } else { @@ -231,9 +234,10 @@ public Partition ensureListPartitionExists(String name, String in) { Objects.requireNonNull(name, "LIST Partition's \"name\" must not be null"); Objects.requireNonNull(in, "LIST Partition's \"in\" must not be null"); Preconditions.checkState(this.partitionType == PartitionType.LIST, "ensureListPartitionExists(String name, String ... in) can only be called for a LIST partitioned VertexLabel. Found %s", this.partitionType.name()); + this.sqlgGraph.getSqlDialect().validateTableName(name); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createListPartition(name, in)); } else { @@ -257,9 +261,10 @@ public Partition ensureListPartitionWithSubPartitionExists(String name, String i Objects.requireNonNull(partitionType, "Sub-partition's \"partitionType\" must not be null"); Objects.requireNonNull(partitionExpression, "Sub-partition's \"partitionExpression\" must not be null"); Preconditions.checkState(this.partitionType == PartitionType.LIST, "ensureRangePartitionExists(String name, String ... in) can only be called for a LIST partitioned VertexLabel. Found %s", this.partitionType.name()); + this.sqlgGraph.getSqlDialect().validateTableName(name); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createListPartitionWithSubPartition(name, in, partitionType, partitionExpression)); } else { @@ -287,7 +292,7 @@ public Partition ensureHashPartitionWithSubPartitionExists(String name, Integer Preconditions.checkState(this.partitionType == PartitionType.HASH, "ensureHashPartitionWithSubPartitionExists can only be called for a HASH partitioned VertexLabel. Found %s", this.partitionType.name()); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - getTopology().lock(); + getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createHashPartitionWithSubPartition(name, modulus, remainder, partitionType, partitionExpression)); } else { @@ -384,7 +389,7 @@ public Index ensureIndexExists(final IndexType indexType, final List indexOptional = this.getIndex(indexName); if (indexOptional.isEmpty()) { - this.getTopology().lock(); + this.getTopology().startSchemaChange(); indexOptional = this.getIndex(indexName); if (indexOptional.isEmpty()) { return this.createIndex(indexName, indexType, properties); @@ -514,7 +519,7 @@ public Map getProperties() { } public ListOrderedSet getIdentifiers() { - ListOrderedSet result = ListOrderedSet.listOrderedSet(this.identifiers); + ListOrderedSet result = ListOrderedSet.listOrderedSet(new ArrayList<>(this.identifiers.asList())); if (getTopology().isSchemaChanged()) { result.addAll(this.uncommittedIdentifiers); for (Pair oldNew : this.renamedIdentifiers) { @@ -766,10 +771,12 @@ void afterCommit() { it.remove(); } this.identifiers.addAll(this.uncommittedIdentifiers); + int index = -1; for (Iterator> it = this.renamedIdentifiers.iterator(); it.hasNext(); ) { + index++; Pair oldName = it.next(); this.identifiers.remove(oldName.getLeft()); - this.identifiers.add(oldName.getRight()); + this.identifiers.add(index, oldName.getRight()); it.remove(); } this.uncommittedIdentifiers.clear(); @@ -1183,7 +1190,7 @@ void renameColumn(String schema, String table, String column, String newName) { * @param preserveData should we keep the SQL data */ void removeIndex(Index idx, boolean preserveData) { - this.getTopology().lock(); + this.getTopology().startSchemaChange(); if (!uncommittedRemovedIndexes.contains(idx.getName())) { uncommittedRemovedIndexes.add(idx.getName()); TopologyManager.removeIndex(this.sqlgGraph, idx); @@ -1205,7 +1212,7 @@ boolean isValid() { } public void removePartition(Partition partition, boolean preserveData) { - this.getTopology().lock(); + this.getTopology().startSchemaChange(); for (Partition partition1 : partition.getPartitions().values()) { partition1.remove(preserveData); @@ -1281,7 +1288,7 @@ public void ensureDistributed(int shardCount, PropertyColumn distributionPropert Preconditions.checkState(getProperty(distributionPropertyColumn.getName()).get().equals(distributionPropertyColumn), "distributionPropertyColumn \"%s\" must be a property of \"%s\"", distributionPropertyColumn.getName(), this.getFullName()); Preconditions.checkArgument(getIdentifiers().contains(distributionPropertyColumn.getName()), "The distribution column must be part of the primary key"); if (!this.isDistributed()) { - this.getTopology().lock(); + this.getTopology().startSchemaChange(); if (!this.isDistributed()) { TopologyManager.distributeAbstractLabel(this.sqlgGraph, this, shardCount, distributionPropertyColumn, colocate); distribute(shardCount, distributionPropertyColumn, colocate); diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/EdgeLabel.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/EdgeLabel.java index 49fcaa7c9..fc1cd367b 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/EdgeLabel.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/EdgeLabel.java @@ -14,6 +14,7 @@ import org.umlg.sqlg.sql.dialect.SqlDialect; import org.umlg.sqlg.structure.PropertyType; import org.umlg.sqlg.structure.SchemaTable; +import org.umlg.sqlg.structure.SqlgGraph; import org.umlg.sqlg.structure.TopologyChangeAction; import org.umlg.sqlg.util.ThreadLocalSet; @@ -156,6 +157,11 @@ private EdgeLabel( this.topology = outVertexLabel.getSchema().getTopology(); } + EdgeLabel(SqlgGraph sqlgGraph, String edgeLabelName, Map properties, ListOrderedSet identifiers) { + super(sqlgGraph, edgeLabelName, properties, identifiers); + this.topology = sqlgGraph.getTopology(); + } + EdgeLabel(Topology topology, String edgeLabelName) { super(topology.getSqlgGraph(), edgeLabelName, Collections.emptyMap(), new ListOrderedSet<>()); this.topology = topology; @@ -197,7 +203,7 @@ public void ensurePropertiesExist(Map columns) { Preconditions.checkState(!this.getSchema().isSqlgSchema(), "schema may not be %s", SQLG_SCHEMA); this.sqlgGraph.getSqlDialect().validateColumnName(column.getKey()); if (!this.uncommittedProperties.containsKey(column.getKey())) { - this.getSchema().getTopology().lock(); + this.getSchema().getTopology().startSchemaChange(); if (getProperty(column.getKey()).isEmpty()) { TopologyManager.addEdgeColumn(this.sqlgGraph, this.getSchema().getName(), EDGE_PREFIX + getLabel(), column, new ListOrderedSet<>()); addColumn(this.getSchema().getName(), EDGE_PREFIX + getLabel(), ImmutablePair.of(column.getKey(), column.getValue())); @@ -797,7 +803,7 @@ public void ensureEdgeVertexLabelExist(Direction direction, VertexLabel vertexLa if (!foreignKeysContains(direction, vertexLabel)) { //Make sure the current thread/transaction owns the lock Schema schema = this.getSchema(); - schema.getTopology().lock(); + schema.getTopology().startSchemaChange(); if (!foreignKeysContains(direction, vertexLabel)) { SchemaTable foreignKeySchemaTable = SchemaTable.of(vertexLabel.getSchema().getName(), vertexLabel.getLabel()); TopologyManager.addLabelToEdge(this.sqlgGraph, this.getSchema().getName(), EDGE_PREFIX + getLabel(), direction == Direction.IN, foreignKeySchemaTable); @@ -1175,7 +1181,7 @@ public String getPrefix() { @Override void removeProperty(PropertyColumn propertyColumn, boolean preserveData) { - this.getSchema().getTopology().lock(); + this.getSchema().getTopology().startSchemaChange(); if (!uncommittedRemovedProperties.contains(propertyColumn.getName())) { uncommittedRemovedProperties.add(propertyColumn.getName()); TopologyManager.removeEdgeColumn(this.sqlgGraph, this.getSchema().getName(), EDGE_PREFIX + getLabel(), propertyColumn.getName()); @@ -1188,13 +1194,21 @@ void removeProperty(PropertyColumn propertyColumn, boolean preserveData) { @Override void renameProperty(String name, PropertyColumn propertyColumn) { - Pair namePair = Pair.of(propertyColumn.getName(), name); -// if (!this.uncommittedRenamedProperties.contains(namePair)) { -// this.uncommittedRenamedProperties.add(namePair); -// TopologyManager.renamePropertyColumn(this.sqlgGraph, getSchema().getName(), EDGE_PREFIX + getLabel(), propertyColumn.getName(), name); -// renameColumn(getSchema().getName(), EDGE_PREFIX + getLabel(), propertyColumn.getName(), name); -// this.getSchema().getTopology().fire(propertyColumn, namePair.getLeft(), TopologyChangeAction.UPDATE); -// } + this.getSchema().getTopology().startSchemaChange(); + String oldName = propertyColumn.getName(); + Pair namePair = Pair.of(oldName, name); + if (!this.uncommittedRemovedProperties.contains(name)) { + this.uncommittedRemovedProperties.add(oldName); + PropertyColumn copy = new PropertyColumn(this, name, propertyColumn.getPropertyType()); + this.uncommittedProperties.put(name, copy); + TopologyManager.renameEdgeLabelPropertyColumn(this.sqlgGraph, getSchema().getName(), EDGE_PREFIX + getLabel(), oldName, name); + renameColumn(getSchema().getName(), EDGE_PREFIX + getLabel(), oldName, name); + if (this.getIdentifiers().contains(oldName)) { + Preconditions.checkState(!this.renamedIdentifiers.contains(namePair), "BUG! renamedIdentifiers may not yet contain '%s'", oldName); + this.renamedIdentifiers.add(namePair); + } + this.getSchema().getTopology().fire(copy, propertyColumn, TopologyChangeAction.UPDATE); + } } @Override @@ -1298,7 +1312,7 @@ EdgeLabel readOnlyCopy(Topology topology, Schema foreignSchema, Set fore } for (VertexLabel inVertexLabel : this.inVertexLabels) { Optional foreignInVertexLabelOptional = foreignSchemas.stream() - .filter(s -> s.getVertexLabel(inVertexLabel.getLabel()).isPresent()) + .filter(s -> s.getName().equals(inVertexLabel.getSchema().getName()) && s.getVertexLabel(inVertexLabel.getLabel()).isPresent()) .map(s -> s.getVertexLabel(inVertexLabel.getLabel()).orElseThrow()) .findAny(); Preconditions.checkState(foreignInVertexLabelOptional.isPresent()); @@ -1309,30 +1323,133 @@ EdgeLabel readOnlyCopy(Topology topology, Schema foreignSchema, Set fore for (String property : this.properties.keySet()) { copy.properties.put(property, this.properties.get(property).readOnlyCopy(copy)); } + copy.identifiers.addAll(this.identifiers); return copy; } - void renameOutVertexLabel(VertexLabel renamedVertexLabel, VertexLabel oldVertexLabel) { - this.uncommittedRemovedOutVertexLabels.add(oldVertexLabel); - this.uncommittedOutVertexLabels.add(renamedVertexLabel); - renamedVertexLabel.addToUncommittedOutEdgeLabels(renamedVertexLabel.getSchema(), this); + void renameOutForeignKeyIdentifier(String oldName, String newName, VertexLabel oldVertexLabel) { + renameInOutForeignKeyIdentifier(oldName, newName, oldVertexLabel, Direction.OUT); + } + + void renameInForeignKeyIdentifier(String oldName, String newName, VertexLabel oldVertexLabel) { + renameInOutForeignKeyIdentifier(oldName, newName, oldVertexLabel, Direction.IN); + } + + private void renameInOutForeignKeyIdentifier(String oldName, String newName, VertexLabel oldVertexLabel, Direction direction) { + Preconditions.checkState(!oldVertexLabel.hasIDPrimaryKey()); + Optional newIdentifierOptional = oldVertexLabel.getIdentifiers().stream().filter(i -> i.equals(newName)).findAny(); + Preconditions.checkState(newIdentifierOptional.isPresent()); + Preconditions.checkState(oldVertexLabel.renamedIdentifiers.stream().anyMatch(p -> p.getLeft().equals(oldName))); renameColumn( getSchema().getName(), EDGE_PREFIX + getLabel(), - oldVertexLabel.getFullName() + Topology.OUT_VERTEX_COLUMN_END, - renamedVertexLabel.getFullName() + Topology.OUT_VERTEX_COLUMN_END + oldVertexLabel.getFullName() + "." + oldName + (direction == Direction.OUT ? Topology.OUT_VERTEX_COLUMN_END : IN_VERTEX_COLUMN_END), + oldVertexLabel.getFullName() + "." + newName + (direction == Direction.OUT ? Topology.OUT_VERTEX_COLUMN_END : IN_VERTEX_COLUMN_END) ); } + void renameOutVertexLabel(VertexLabel renamedVertexLabel, VertexLabel oldVertexLabel) { + this.uncommittedRemovedOutVertexLabels.add(oldVertexLabel); + this.uncommittedOutVertexLabels.add(renamedVertexLabel); + renamedVertexLabel.addToUncommittedOutEdgeLabels(renamedVertexLabel.getSchema(), this); + if (oldVertexLabel.hasIDPrimaryKey()) { + renameColumn( + getSchema().getName(), + EDGE_PREFIX + getLabel(), + oldVertexLabel.getFullName() + Topology.OUT_VERTEX_COLUMN_END, + renamedVertexLabel.getFullName() + Topology.OUT_VERTEX_COLUMN_END + ); + } else { + for (String identifier : oldVertexLabel.getIdentifiers()) { + renameColumn( + getSchema().getName(), + EDGE_PREFIX + getLabel(), + oldVertexLabel.getFullName() + "." + identifier + Topology.OUT_VERTEX_COLUMN_END, + renamedVertexLabel.getFullName() + "." + identifier + Topology.OUT_VERTEX_COLUMN_END + ); + } + } + } + void renameInVertexLabel(VertexLabel renamedVertexLabel, VertexLabel oldVertexLabel) { this.uncommittedRemovedInVertexLabels.add(oldVertexLabel); this.uncommittedInVertexLabels.add(renamedVertexLabel); renamedVertexLabel.addToUncommittedInEdgeLabels(renamedVertexLabel.getSchema(), this); - renameColumn( + if (oldVertexLabel.hasIDPrimaryKey()) { + renameColumn( + getSchema().getName(), + EDGE_PREFIX + getLabel(), + oldVertexLabel.getFullName() + Topology.IN_VERTEX_COLUMN_END, + renamedVertexLabel.getFullName() + Topology.IN_VERTEX_COLUMN_END + ); + } else { + for (String identifier : oldVertexLabel.getIdentifiers()) { + renameColumn( + getSchema().getName(), + EDGE_PREFIX + getLabel(), + oldVertexLabel.getFullName() + "." + identifier + Topology.IN_VERTEX_COLUMN_END, + renamedVertexLabel.getFullName() + "." + identifier + Topology.IN_VERTEX_COLUMN_END + ); + } + } + } + + @Override + public void rename(String label) { + Objects.requireNonNull(label, "Given label must not be null"); + Preconditions.checkArgument(!label.startsWith(EDGE_PREFIX), "label may not be prefixed with \"%s\"", EDGE_PREFIX); + Preconditions.checkState(!this.isForeignAbstractLabel, "'%s' is a read only foreign table!", label); + this.getSchema().getTopology().startSchemaChange(); + this.getSchema().renameEdgeLabel(this, label); + } + + static EdgeLabel renameEdgeLabel( + SqlgGraph sqlgGraph, + Schema schema, + EdgeLabel oldEdgeLabel, + String newLabel, + Set outVertexLabels, + Set inVertexLabels, + Map properties, + ListOrderedSet identifiers) { + + Preconditions.checkArgument(!schema.isSqlgSchema(), "renameEdgeLabel may not be called for \"%s\"", SQLG_SCHEMA); + EdgeLabel edgeLabel = new EdgeLabel( + sqlgGraph, + newLabel, + properties, + identifiers + ); + edgeLabel.uncommittedOutVertexLabels.addAll(outVertexLabels); + edgeLabel.uncommittedInVertexLabels.addAll(inVertexLabels); + edgeLabel.renameEdgeLabelOnDb(oldEdgeLabel.getLabel(), newLabel); + TopologyManager.renameEdgeLabel(sqlgGraph, schema.getName(), EDGE_PREFIX + oldEdgeLabel.getLabel(), EDGE_PREFIX + newLabel); + edgeLabel.committed = false; + for (VertexLabel outVertexLabel : outVertexLabels) { + outVertexLabel.removeOutEdge(oldEdgeLabel); + outVertexLabel.addToUncommittedOutEdgeLabels(edgeLabel.getSchema(), edgeLabel); + } + for (VertexLabel inVertexLabel : inVertexLabels) { + inVertexLabel.removeInEdge(oldEdgeLabel); + inVertexLabel.addToUncommittedInEdgeLabels(edgeLabel.getSchema(), edgeLabel); + } + return edgeLabel; + } + + private void renameEdgeLabelOnDb(String oldLabel, String newLabel) { + String sql = this.sqlgGraph.getSqlDialect().renameTable( getSchema().getName(), - EDGE_PREFIX + getLabel(), - oldVertexLabel.getFullName() + Topology.IN_VERTEX_COLUMN_END, - renamedVertexLabel.getFullName() + Topology.IN_VERTEX_COLUMN_END + EDGE_PREFIX + oldLabel, + EDGE_PREFIX + newLabel ); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(sql); + } + Connection conn = this.sqlgGraph.tx().getConnection(); + try (Statement stmt = conn.createStatement()) { + stmt.execute(sql); + } catch (SQLException e) { + throw new RuntimeException(e); + } } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Partition.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Partition.java index 5c0b384b6..53629d453 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Partition.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Partition.java @@ -214,7 +214,7 @@ public void remove(boolean preserveData) { } private void removePartition(Partition partition, boolean preserveData) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); for (Partition partition1 : partition.getPartitions().values()) { partition.removePartition(partition1, preserveData); } @@ -1092,7 +1092,7 @@ public void ensureRangePartitionExists(String name, String from, String to) { Preconditions.checkState(this.partitionType == PartitionType.RANGE, "ensureRangePartitionExists(String name, String from, String to) can only be called for a RANGE partitioned VertexLabel. Found %s", this.partitionType.name()); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); partitionOptional.orElseGet(() -> this.createRangePartition(name, from, to)); } @@ -1104,7 +1104,7 @@ public void ensureListPartitionExists(String name, String in) { Preconditions.checkState(this.partitionType == PartitionType.LIST, "ensureListPartitionExists(String name, String in) can only be called for a LIST partitioned VertexLabel. Found %s", this.partitionType.name()); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); partitionOptional.orElseGet(() -> this.createListPartition(name, in)); } @@ -1117,7 +1117,7 @@ public void ensureHashPartitionExists(String name, Integer modulus, Integer rema Preconditions.checkState(this.partitionType == PartitionType.HASH, "ensureHashPartitionExists(String name, String in) can only be called for a HASH partitioned VertexLabel. Found %s", this.partitionType.name()); Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); partitionOptional.orElseGet(() -> this.createHashPartition(name, modulus, remainder)); } @@ -1138,7 +1138,7 @@ public Partition ensureRangePartitionWithSubPartitionExists( Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createRangePartitionWithSubPartition(name, from, to, partitionType, partitionExpression)); } else { @@ -1159,7 +1159,7 @@ public Partition ensureListPartitionWithSubPartitionExists( Optional partitionOptional = this.getPartition(name); if (partitionOptional.isEmpty()) { - this.getAbstractLabel().getSchema().getTopology().lock(); + this.getAbstractLabel().getSchema().getTopology().startSchemaChange(); partitionOptional = this.getPartition(name); return partitionOptional.orElseGet(() -> this.createListPartitionWithSubPartition(name, in, partitionType, partitionExpression)); } else { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Schema.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Schema.java index 818554629..37f046e8a 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Schema.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Schema.java @@ -152,7 +152,7 @@ void ensureTemporaryVertexTableExist(final String label, final Map columns, ListOrderedSet identifiers) { Objects.requireNonNull(label, "Given table must not be null"); Preconditions.checkArgument(!label.startsWith(VERTEX_PREFIX), "label may not be prefixed with \"%s\"", VERTEX_PREFIX); - Preconditions.checkState(!this.isForeignSchema, "'%s' is a read only foreign schema!", this.name); for (String identifier : identifiers) { Preconditions.checkState(columns.containsKey(identifier), "The identifiers must be in the specified columns. \"%s\" not found", identifier); } Optional vertexLabelOptional = this.getVertexLabel(label); if (vertexLabelOptional.isEmpty()) { - this.topology.lock(); + Preconditions.checkState(!this.isForeignSchema, "'%s' is a read only foreign schema!", this.name); + this.topology.startSchemaChange(); vertexLabelOptional = this.getVertexLabel(label); if (vertexLabelOptional.isEmpty()) { return this.createVertexLabel(label, columns, identifiers); @@ -192,7 +192,7 @@ public VertexLabel ensureVertexLabelExist(final String label, final Map vertexLabelOptional = this.getVertexLabel(label); Preconditions.checkState(vertexLabelOptional.isEmpty(), "'%s' already exists", label); - Preconditions.checkState(!this.isSqlgSchema(), "createVertexLabel may not be called for \"%s\"", SQLG_SCHEMA); + Preconditions.checkState(!this.isSqlgSchema(), "renameVertexLabel may not be called for \"%s\"", SQLG_SCHEMA); Preconditions.checkArgument(!label.startsWith(VERTEX_PREFIX), "vertex label may not start with " + VERTEX_PREFIX); this.sqlgGraph.getSqlDialect().validateTableName(label); this.uncommittedRemovedVertexLabels.add(this.name + "." + VERTEX_PREFIX + vertexLabel.label); @@ -209,6 +209,32 @@ VertexLabel renameVertexLabel(VertexLabel vertexLabel, String label) { return renamedVertexLabel; } + @SuppressWarnings("UnusedReturnValue") + void renameEdgeLabel(EdgeLabel edgeLabel, String label) { + Optional edgeLabelOptional = this.getEdgeLabel(label); + Preconditions.checkState(edgeLabelOptional.isEmpty(), "'%s' already exists", label); + Preconditions.checkState(!this.isSqlgSchema(), "renameEdgeLabel may not be called for \"%s\"", SQLG_SCHEMA); + Preconditions.checkArgument(!label.startsWith(EDGE_PREFIX), "edge label may not start with " + EDGE_PREFIX); + this.sqlgGraph.getSqlDialect().validateTableName(label); + this.uncommittedRemovedEdgeLabels.add(this.name + "." + EDGE_PREFIX + edgeLabel.label); + + Set outVertexLabels = edgeLabel.getOutVertexLabels(); + Set inVertexLabels = edgeLabel.getInVertexLabels(); + + EdgeLabel renamedEdgeLabel = EdgeLabel.renameEdgeLabel( + this.sqlgGraph, + this, + edgeLabel, + label, + outVertexLabels, + inVertexLabels, + edgeLabel.getPropertyTypeMap(), + edgeLabel.getIdentifiers() + ); + this.uncommittedOutEdgeLabels.put(this.name + "." + EDGE_PREFIX + label, renamedEdgeLabel); + this.getTopology().fire(renamedEdgeLabel, edgeLabel, TopologyChangeAction.UPDATE); + } + public VertexLabel ensurePartitionedVertexLabelExist( final String label, final Map columns, @@ -233,7 +259,7 @@ public VertexLabel ensurePartitionedVertexLabelExist( Optional vertexLabelOptional = this.getVertexLabel(label); if (vertexLabelOptional.isEmpty()) { - this.topology.lock(); + this.topology.startSchemaChange(); vertexLabelOptional = this.getVertexLabel(label); return vertexLabelOptional.orElseGet( () -> this.createPartitionedVertexLabel(label, columns, identifiers, partitionType, partitionExpression, addPrimaryKeyConstraint) @@ -261,7 +287,6 @@ public EdgeLabel ensureEdgeLabelExist( Map columns, ListOrderedSet identifiers) { - Preconditions.checkState(!this.isForeignSchema, "'A' is a read only foreign schema!"); Objects.requireNonNull(edgeLabelName, "Given edgeLabelName may not be null"); Objects.requireNonNull(outVertexLabel, "Given outVertexLabel may not be null"); Objects.requireNonNull(inVertexLabel, "Given inVertexLabel may not be null"); @@ -284,7 +309,8 @@ public EdgeLabel ensureEdgeLabelExist( EdgeLabel edgeLabel; Optional edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { - this.topology.lock(); + Preconditions.checkState(!this.isForeignSchema, "'A' is a read only foreign schema!"); + this.topology.startSchemaChange(); edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { edgeLabel = this.createEdgeLabel(edgeLabelName, outVertexLabel, inVertexLabel, columns, identifiers); @@ -327,7 +353,7 @@ public EdgeLabel ensureEdgeLabelExist( EdgeLabel edgeLabel; Optional edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { - this.topology.lock(); + this.topology.startSchemaChange(); edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { edgeLabel = this.createEdgeLabel(edgeLabelName, outVertexLabel, inVertexLabel, columns, identifiers); @@ -442,7 +468,7 @@ public EdgeLabel ensurePartitionedEdgeLabelExist( EdgeLabel edgeLabel; Optional edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { - this.topology.lock(); + this.topology.startSchemaChange(); edgeLabelOptional = this.getEdgeLabel(edgeLabelName); if (edgeLabelOptional.isEmpty()) { edgeLabel = this.createPartitionedEdgeLabel( @@ -1681,7 +1707,7 @@ public void remove(boolean preserveData) { * @param preserveData should we keep the SQL data */ void removeEdgeLabel(EdgeLabel edgeLabel, boolean preserveData) { - getTopology().lock(); + getTopology().startSchemaChange(); String fn = this.name + "." + EDGE_PREFIX + edgeLabel.getName(); if (!this.uncommittedRemovedEdgeLabels.contains(fn)) { @@ -1708,7 +1734,7 @@ void removeEdgeLabel(EdgeLabel edgeLabel, boolean preserveData) { * @param preserveData should we keep the SQL data */ void removeVertexLabel(VertexLabel vertexLabel, boolean preserveData) { - getTopology().lock(); + getTopology().startSchemaChange(); String fn = this.name + "." + VERTEX_PREFIX + vertexLabel.getName(); if (!this.uncommittedRemovedVertexLabels.contains(fn)) { this.sqlgGraph.traversal().V().hasLabel(this.name + "." + vertexLabel.getLabel()).drop().iterate(); @@ -1830,7 +1856,7 @@ void readOnlyCopyEdgeLabels(Topology topology, Schema foreignSchema, Set EdgeLabel edgeLabel = this.outEdgeLabels.get(label); EdgeLabel foreignEdgeLabel = edgeLabel.readOnlyCopy(topology, foreignSchema, foreignSchemas); foreignSchema.addToAllEdgeCache(foreignEdgeLabel); - this.outEdgeLabels.put(label, foreignEdgeLabel); + foreignSchema.outEdgeLabels.put(label, foreignEdgeLabel); } } diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Topology.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Topology.java index b97ad4fac..9f718faa2 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Topology.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/Topology.java @@ -23,6 +23,8 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; /** * Date: 2016/09/04 @@ -59,6 +61,8 @@ public class Topology { private final Map schemas = new ConcurrentHashMap<>(); private final ThreadLocal schemaChanged = ThreadLocal.withInitial(() -> false); + private boolean locked = false; + private final ReentrantLock topologyLock = new ReentrantLock(); private final ThreadLocalMap uncommittedSchemas = new ThreadLocalMap<>(); private final Set uncommittedRemovedSchemas = new ConcurrentSkipListSet<>(); private final Map metaSchemas; @@ -511,14 +515,28 @@ public void threadWriteLock() { } /** - * Global lock on the topology. - * For distributed graph (multiple jvm) this happens on the db via a lock sql statement. + * Global indicator to change the topology. */ - void lock() { + void startSchemaChange() { + if (this.locked && this.sqlgGraph.tx().isTopologyLocked()) { + throw new IllegalStateException("The topology is locked! Changes are not allowed, first unlock it. Either globally or for the transaction."); + } this.sqlgGraph.tx().readWrite(); this.schemaChanged.set(true); } + public void lock() { + this.locked = true; + } + + public void unlock() { + this.locked = false; + } + + public boolean isLocked() { + return this.locked; + } + boolean isSchemaChanged() { return this.schemaChanged.get(); } @@ -541,7 +559,7 @@ public Schema ensureSchemaExist(final String schemaName) { Optional schemaOptional = this.getSchema(schemaName); Schema schema; if (schemaOptional.isEmpty()) { - this.lock(); + this.startSchemaChange(); //search again after the lock is obtained. schemaOptional = this.getSchema(schemaName); if (schemaOptional.isEmpty()) { @@ -566,6 +584,7 @@ public Schema ensureSchemaExist(final String schemaName) { */ public void importForeignSchemas(Set originalSchemas) { Preconditions.checkState(!isSchemaChanged(), "To import a foreign schema there must not be any pending changes!"); + Preconditions.checkState(!this.locked, "The topology is locked, first unlock it before importing foreign schemas."); //validate all edge's vertices are in a foreign schema Schema.validateImportingEdgeLabels(originalSchemas); @@ -613,43 +632,45 @@ public void importForeignSchemas(Set originalSchemas) { } } - public void clearForeignSchemas() { + public void clearForeignSchemas(Set schemasToClear) { Set toRemove = new HashSet<>(); - for (Map.Entry schemaEntry : this.schemas.entrySet()) { - String schemaKey = schemaEntry.getKey(); - Schema schema = schemaEntry.getValue(); - if (schema.isForeignSchema()) { - toRemove.add(schemaKey); - for (Map.Entry edgeLabelEntry : schema.getEdgeLabels().entrySet()) { - String key = edgeLabelEntry.getKey(); - EdgeLabel edgeLabel = edgeLabelEntry.getValue(); - Preconditions.checkState(edgeLabel.isForeign()); - Preconditions.checkState(this.allTableCache.remove(key) != null, "Failed to remove '%s' from 'allTableCache'", key); - Preconditions.checkState( - this.edgeForeignKeyCache.remove(schemaKey + "." + EDGE_PREFIX + edgeLabel.getLabel()) != null, - "Failed to remove '%s' from 'edgeForeignKeyCache'", key); - } - for (Map.Entry vertexLabelEntry : schema.getVertexLabels().entrySet()) { - String key = vertexLabelEntry.getKey(); - VertexLabel vertexLabel = vertexLabelEntry.getValue(); - Preconditions.checkState(vertexLabel.isForeign()); - Preconditions.checkState(this.allTableCache.remove(key) != null, "Failed to remove '%s' from 'allTableCache'", key); - SchemaTable schemaTable = SchemaTable.of(schemaKey, VERTEX_PREFIX + vertexLabel.getLabel()); - Preconditions.checkState(this.schemaTableForeignKeyCache.remove(schemaTable) != null, "Failed to remove '%s' from 'schemaTableForeignKeyCache'", key); - } - } else { - Pair>, Set>> removed = schema.clearForeignAbstractLabels(); - for (Pair vertex: removed.getLeft()) { - Preconditions.checkState(this.allTableCache.remove(vertex.getLeft()) != null, "Failed to remove '%s' from 'allTableCache", vertex.getLeft()); - SchemaTable schemaTable = SchemaTable.of(schemaKey, VERTEX_PREFIX + vertex.getRight()); - Preconditions.checkState(this.schemaTableForeignKeyCache.remove(schemaTable) != null, "Failed to remove '%s' from 'schemaTableForeignKeyCache'", schemaTable.toString()); - } - for (Pair edge: removed.getRight()) { - Preconditions.checkState(this.allTableCache.remove(edge.getLeft()) != null, "Failed to remove '%s' from 'allTableCache", edge.getLeft()); - Preconditions.checkState( - this.edgeForeignKeyCache.remove(schemaKey + "." + EDGE_PREFIX + edge.getRight()) != null, - "Failed to remove '%s' from 'edgeForeignKeyCache'", edge); + for (String schemaNameToRemove : schemasToClear) { + if (this.schemas.containsKey(schemaNameToRemove)) { + Schema schema = this.schemas.get(schemaNameToRemove); + if (schema.isForeignSchema()) { + toRemove.add(schemaNameToRemove); + for (Map.Entry edgeLabelEntry : schema.getEdgeLabels().entrySet()) { + String key = edgeLabelEntry.getKey(); + EdgeLabel edgeLabel = edgeLabelEntry.getValue(); + Preconditions.checkState(edgeLabel.isForeign()); + Preconditions.checkState(this.allTableCache.remove(key) != null, "Failed to remove '%s' from 'allTableCache'", key); + Preconditions.checkState( + this.edgeForeignKeyCache.remove(schemaNameToRemove + "." + EDGE_PREFIX + edgeLabel.getLabel()) != null, + "Failed to remove '%s' from 'edgeForeignKeyCache'", key); + } + for (Map.Entry vertexLabelEntry : schema.getVertexLabels().entrySet()) { + String key = vertexLabelEntry.getKey(); + VertexLabel vertexLabel = vertexLabelEntry.getValue(); + Preconditions.checkState(vertexLabel.isForeign()); + Preconditions.checkState(this.allTableCache.remove(key) != null, "Failed to remove '%s' from 'allTableCache'", key); + SchemaTable schemaTable = SchemaTable.of(schemaNameToRemove, VERTEX_PREFIX + vertexLabel.getLabel()); + Preconditions.checkState(this.schemaTableForeignKeyCache.remove(schemaTable) != null, "Failed to remove '%s' from 'schemaTableForeignKeyCache'", key); + } + } else { + Pair>, Set>> removed = schema.clearForeignAbstractLabels(); + for (Pair vertex: removed.getLeft()) { + Preconditions.checkState(this.allTableCache.remove(vertex.getLeft()) != null, "Failed to remove '%s' from 'allTableCache", vertex.getLeft()); + SchemaTable schemaTable = SchemaTable.of(schemaNameToRemove, VERTEX_PREFIX + vertex.getRight()); + Preconditions.checkState(this.schemaTableForeignKeyCache.remove(schemaTable) != null, "Failed to remove '%s' from 'schemaTableForeignKeyCache'", schemaTable.toString()); + } + for (Pair edge: removed.getRight()) { + Preconditions.checkState(this.allTableCache.remove(edge.getLeft()) != null, "Failed to remove '%s' from 'allTableCache", edge.getLeft()); + Preconditions.checkState( + this.edgeForeignKeyCache.remove(schemaNameToRemove + "." + EDGE_PREFIX + edge.getRight()) != null, + "Failed to remove '%s' from 'edgeForeignKeyCache'", edge); + } } + } } for (String remove : toRemove) { @@ -657,6 +678,10 @@ public void clearForeignSchemas() { } } + public void clearForeignSchemas() { + clearForeignSchemas(this.schemas.values().stream().map(Schema::getName).collect(Collectors.toSet())); + } + public void importForeignVertexEdgeLabels(Schema importIntoSchema, Set vertexLabels, Set edgeLabels) { importIntoSchema.importForeignVertexAndEdgeLabels(vertexLabels, edgeLabels); for (VertexLabel vertexLabel : vertexLabels) { @@ -1027,7 +1052,7 @@ private void deallocateAll() { } public void cacheTopology() { - this.lock(); + this.startSchemaChange(); GraphTraversalSource traversalSource = this.sqlgGraph.topology(); //load the last log //the last timestamp is needed when just after obtaining the lock the log table is queried again to ensure that the last log is indeed @@ -1692,7 +1717,7 @@ void fire(TopologyInf topologyInf, TopologyInf oldValue, TopologyChangeAction ac * @param preserveData should we preserve the SQL data? */ void removeSchema(Schema schema, boolean preserveData) { - lock(); + startSchemaChange(); if (!this.uncommittedRemovedSchemas.contains(schema.getName())) { // remove edge roles in other schemas pointing to vertex labels in removed schema // TODO undo this in case of rollback? diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/TopologyManager.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/TopologyManager.java index f5059bb7b..5c873f3c6 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/TopologyManager.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/TopologyManager.java @@ -229,6 +229,25 @@ public static void renameVertexLabel(SqlgGraph sqlgGraph, String schema, String } } + public static void renameEdgeLabel(SqlgGraph sqlgGraph, String schema, String oldEdgeLabel, String newEdgeLabel) { + BatchManager.BatchModeType batchModeType = flushAndSetTxToNone(sqlgGraph); + try { + Preconditions.checkArgument(oldEdgeLabel.startsWith(EDGE_PREFIX), "prefixedTable must be for a edge. prefixedTable = " + oldEdgeLabel); + GraphTraversalSource traversalSource = sqlgGraph.topology(); + List edgeLabelsToRename = traversalSource.V() + .hasLabel(SQLG_SCHEMA + "." + SQLG_SCHEMA_SCHEMA) + .has(SQLG_SCHEMA_SCHEMA_NAME, schema) + .out(SQLG_SCHEMA_SCHEMA_VERTEX_EDGE) + .out(SQLG_SCHEMA_OUT_EDGES_EDGE) + .has(SQLG_SCHEMA_EDGE_LABEL_NAME, oldEdgeLabel.substring(EDGE_PREFIX.length())) + .toList(); + Preconditions.checkState(edgeLabelsToRename.size() == 1, String.format("Expected exactly one VertexLabel in %s.%s. Found %d", schema, oldEdgeLabel, edgeLabelsToRename.size())); + edgeLabelsToRename.get(0).property(SQLG_SCHEMA_EDGE_LABEL_NAME, newEdgeLabel.substring(EDGE_PREFIX.length())); + } finally { + sqlgGraph.tx().batchMode(batchModeType); + } + } + public static void removeVertexLabel(SqlgGraph sqlgGraph, VertexLabel vertexLabel) { BatchManager.BatchModeType batchModeType = flushAndSetTxToNone(sqlgGraph); try { @@ -1005,7 +1024,7 @@ public static void removeVertexColumn(SqlgGraph sqlgGraph, String schema, String } - public static void renamePropertyColumn(SqlgGraph sqlgGraph, String schema, String prefixedTable, String column, String newName) { + static void renameVertexLabelPropertyColumn(SqlgGraph sqlgGraph, String schema, String prefixedTable, String column, String newName) { BatchManager.BatchModeType batchModeType = flushAndSetTxToNone(sqlgGraph); try { Preconditions.checkArgument(prefixedTable.startsWith(VERTEX_PREFIX), "prefixedTable must be for a vertex. prefixedTable = " + prefixedTable); @@ -1027,6 +1046,28 @@ public static void renamePropertyColumn(SqlgGraph sqlgGraph, String schema, Stri } } + static void renameEdgeLabelPropertyColumn(SqlgGraph sqlgGraph, String schema, String prefixedTable, String column, String newName) { + BatchManager.BatchModeType batchModeType = flushAndSetTxToNone(sqlgGraph); + try { + Preconditions.checkArgument(prefixedTable.startsWith(EDGE_PREFIX), "prefixedTable must be for a edge. prefixedTable = " + prefixedTable); + GraphTraversalSource traversalSource = sqlgGraph.topology(); + + List propertiesToRename = traversalSource.V() + .hasLabel(SQLG_SCHEMA + "." + SQLG_SCHEMA_SCHEMA) + .has(SQLG_SCHEMA_SCHEMA_NAME, schema) + .out(SQLG_SCHEMA_SCHEMA_VERTEX_EDGE) + .out(SQLG_SCHEMA_OUT_EDGES_EDGE) + .has(SQLG_SCHEMA_EDGE_LABEL_NAME, prefixedTable.substring(EDGE_PREFIX.length())) + .out(SQLG_SCHEMA_EDGE_PROPERTIES_EDGE) + .has(SQLG_SCHEMA_PROPERTY_NAME, column) + .toList(); + Preconditions.checkState(propertiesToRename.size() == 1, String.format("Expected exactly one property in %s.%s.%s. Found %d", schema, propertiesToRename, column, propertiesToRename.size())); + propertiesToRename.get(0).property(SQLG_SCHEMA_PROPERTY_NAME, newName); + } finally { + sqlgGraph.tx().batchMode(batchModeType); + } + } + public static void removeEdgeColumn(SqlgGraph sqlgGraph, String schema, String prefixedTable, String column) { BatchManager.BatchModeType batchModeType = flushAndSetTxToNone(sqlgGraph); try { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/VertexLabel.java b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/VertexLabel.java index 9a953dcc1..4e96f5d70 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/VertexLabel.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/structure/topology/VertexLabel.java @@ -75,7 +75,14 @@ static VertexLabel createVertexLabel(SqlgGraph sqlgGraph, Schema schema, String return vertexLabel; } - static VertexLabel renameVertexLabel(SqlgGraph sqlgGraph, Schema schema, String oldLabel, String newLabel, Map columns, ListOrderedSet identifiers) { + static VertexLabel renameVertexLabel( + SqlgGraph sqlgGraph, + Schema schema, + String oldLabel, + String newLabel, + Map columns, + ListOrderedSet identifiers) { + Preconditions.checkArgument(!schema.isSqlgSchema(), "renameVertexLabel may not be called for \"%s\"", SQLG_SCHEMA); VertexLabel vertexLabel = new VertexLabel(schema, newLabel, columns, identifiers); vertexLabel.renameVertexLabelOnDb(oldLabel, newLabel); @@ -280,14 +287,12 @@ void addToUncommittedOutEdgeLabels(Schema schema, EdgeLabel edgeLabel) { * @param edgeLabelName The edge's label. * @param inVertexLabel The edge's in {@link VertexLabel}. 'this' is the out {@link VertexLabel}. * @param properties A map of the edge's properties. - * @return The {@link EdgeLabel} that been loaded. */ - EdgeLabel loadSqlgSchemaEdgeLabel(String edgeLabelName, VertexLabel inVertexLabel, Map properties) { + void loadSqlgSchemaEdgeLabel(String edgeLabelName, VertexLabel inVertexLabel, Map properties) { Preconditions.checkState(this.schema.isSqlgSchema(), "loadSqlgSchemaEdgeLabel must be called for \"%s\" found \"%s\"", SQLG_SCHEMA, this.schema.getName()); EdgeLabel edgeLabel = EdgeLabel.loadSqlgSchemaEdgeLabel(edgeLabelName, this, inVertexLabel, properties); this.outEdgeLabels.put(this.schema.getName() + "." + edgeLabel.getLabel(), edgeLabel); inVertexLabel.inEdgeLabels.put(this.schema.getName() + "." + edgeLabel.getLabel(), edgeLabel); - return edgeLabel; } /** @@ -444,13 +449,13 @@ EdgeLabel addEdgeLabel( // @Override public void ensurePropertiesExist(Map columns) { - Preconditions.checkState(!this.isForeignAbstractLabel, "'%s' is a read only foreign VertexLabel!", this.label); for (Map.Entry column : columns.entrySet()) { if (!this.properties.containsKey(column.getKey())) { + Preconditions.checkState(!this.isForeignAbstractLabel, "'%s' is a read only foreign VertexLabel!", this.label); Preconditions.checkState(!this.schema.isSqlgSchema(), "schema may not be %s", SQLG_SCHEMA); this.sqlgGraph.getSqlDialect().validateColumnName(column.getKey()); if (!this.uncommittedProperties.containsKey(column.getKey())) { - this.schema.getTopology().lock(); + this.schema.getTopology().startSchemaChange(); if (getProperty(column.getKey()).isEmpty()) { TopologyManager.addVertexColumn(this.sqlgGraph, this.schema.getName(), VERTEX_PREFIX + getLabel(), column); addColumn(this.schema.getName(), VERTEX_PREFIX + getLabel(), ImmutablePair.of(column.getKey(), column.getValue())); @@ -516,21 +521,17 @@ private void createVertexLabelOnDb(Map columns, ListOrdere } private void renameVertexLabelOnDb(String oldLabel, String newLabel) { - StringBuilder sql = new StringBuilder("ALTER TABLE "); - sql.append(this.sqlgGraph.getSqlDialect().maybeWrapInQoutes(this.schema.getName())); - sql.append("."); - sql.append(this.sqlgGraph.getSqlDialect().maybeWrapInQoutes(VERTEX_PREFIX + oldLabel)); - sql.append(" RENAME TO "); - sql.append(this.sqlgGraph.getSqlDialect().maybeWrapInQoutes(VERTEX_PREFIX + newLabel)); - if (this.sqlgGraph.getSqlDialect().needsSemicolon()) { - sql.append(";"); - } + String sql = this.sqlgGraph.getSqlDialect().renameTable( + this.schema.getName(), + VERTEX_PREFIX + oldLabel, + VERTEX_PREFIX + newLabel + ); if (LOGGER.isDebugEnabled()) { - LOGGER.debug(sql.toString()); + LOGGER.debug(sql); } Connection conn = this.sqlgGraph.tx().getConnection(); try (Statement stmt = conn.createStatement()) { - stmt.execute(sql.toString()); + stmt.execute(sql); } catch (SQLException e) { throw new RuntimeException(e); } @@ -1145,7 +1146,7 @@ Pair, Set> getUncommittedRemovedSchemaTableForeign @Override void removeProperty(PropertyColumn propertyColumn, boolean preserveData) { - this.getSchema().getTopology().lock(); + this.getSchema().getTopology().startSchemaChange(); if (!this.uncommittedRemovedProperties.contains(propertyColumn.getName())) { this.uncommittedRemovedProperties.add(propertyColumn.getName()); for (Index index : getIndexes().values()) { @@ -1166,18 +1167,30 @@ void removeProperty(PropertyColumn propertyColumn, boolean preserveData) { @Override void renameProperty(String name, PropertyColumn propertyColumn) { - this.getSchema().getTopology().lock(); + this.getSchema().getTopology().startSchemaChange(); String oldName = propertyColumn.getName(); Pair namePair = Pair.of(oldName, name); if (!this.uncommittedRemovedProperties.contains(name)) { this.uncommittedRemovedProperties.add(oldName); PropertyColumn copy = new PropertyColumn(this, name, propertyColumn.getPropertyType()); this.uncommittedProperties.put(name, copy); - TopologyManager.renamePropertyColumn(this.sqlgGraph, this.schema.getName(), VERTEX_PREFIX + getLabel(), oldName, name); + TopologyManager.renameVertexLabelPropertyColumn(this.sqlgGraph, this.schema.getName(), VERTEX_PREFIX + getLabel(), oldName, name); renameColumn(this.schema.getName(), VERTEX_PREFIX + getLabel(), oldName, name); if (this.getIdentifiers().contains(oldName)) { Preconditions.checkState(!this.renamedIdentifiers.contains(namePair), "BUG! renamedIdentifiers may not yet contain '%s'", oldName); this.renamedIdentifiers.add(namePair); + + Map outEdgeLabels = getOutEdgeLabels(); + for (String outEdgeLabel : outEdgeLabels.keySet()) { + EdgeLabel edgeLabel = outEdgeLabels.get(outEdgeLabel); + edgeLabel.renameOutForeignKeyIdentifier(oldName, name, this); + } + Map inEdgeLabels = getInEdgeLabels(); + for (String inEdgeLabel : inEdgeLabels.keySet()) { + EdgeLabel edgeLabel = inEdgeLabels.get(inEdgeLabel); + edgeLabel.renameInForeignKeyIdentifier(oldName, name, this); + } + } this.getSchema().getTopology().fire(copy, propertyColumn, TopologyChangeAction.UPDATE); } @@ -1222,7 +1235,7 @@ void removeEdgeRole(EdgeRole er, boolean dropEdges, boolean preserveData) { if (ers.size() == 1) { er.getEdgeLabel().remove(preserveData); } else { - getSchema().getTopology().lock(); + getSchema().getTopology().startSchemaChange(); EdgeLabel edgeLabel = er.getEdgeLabel(); switch (er.getDirection()) { // we don't support both @@ -1282,6 +1295,7 @@ VertexLabel readOnlyCopy(Schema schema) { for (String property : this.properties.keySet()) { copy.properties.put(property, this.properties.get(property).readOnlyCopy(copy)); } + copy.identifiers.addAll(this.identifiers); return copy; } @@ -1290,7 +1304,7 @@ public void rename(String label) { Objects.requireNonNull(label, "Given label must not be null"); Preconditions.checkArgument(!label.startsWith(VERTEX_PREFIX), "label may not be prefixed with \"%s\"", VERTEX_PREFIX); Preconditions.checkState(!this.isForeignAbstractLabel, "'%s' is a read only foreign table!", label); - this.getSchema().getTopology().lock(); + this.getSchema().getTopology().startSchemaChange(); VertexLabel renamedVertexLabel = this.getSchema().renameVertexLabel(this, label); Map outEdgeLabels = getOutEdgeLabels(); for (String outEdgeLabel : outEdgeLabels.keySet()) { diff --git a/sqlg-core/src/main/java/org/umlg/sqlg/util/SqlgUtil.java b/sqlg-core/src/main/java/org/umlg/sqlg/util/SqlgUtil.java index f6af21dda..ef423b337 100644 --- a/sqlg-core/src/main/java/org/umlg/sqlg/util/SqlgUtil.java +++ b/sqlg-core/src/main/java/org/umlg/sqlg/util/SqlgUtil.java @@ -179,7 +179,7 @@ private static List> loadLabeledElements( if (!schemaTableTree.getLabels().isEmpty()) { E sqlgElement = null; boolean resultSetWasNull = false; - Long id = -1L; + long id = -1L; if (schemaTableTree.isHasIDPrimaryKey()) { //aggregate queries have no ID if (!schemaTableTree.hasAggregateFunction()) { @@ -427,6 +427,9 @@ public static int setKeyValueAsParameter(SqlgGraph sqlgGraph, boolean mod, int p sqlgGraph.getSqlDialect().setPolygon(preparedStatement, parameterStartIndex, pair.getRight()); parameterStartIndex++; break; + case UUID_ORDINAL: + preparedStatement.setObject(parameterStartIndex++, pair.right); + break; case BOOLEAN_ARRAY_ORDINAL: sqlgGraph.getSqlDialect().setArray(preparedStatement, parameterStartIndex++, PropertyType.BOOLEAN_ARRAY, SqlgUtil.transformArrayToInsertValue(pair.left, pair.right)); break; @@ -1205,6 +1208,12 @@ public static List getValue(ResultSet resultSet, List