From 66d0b9cce194713777c914cded7c241ec786064d Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Sat, 4 Apr 2020 11:23:19 +0200 Subject: [PATCH 1/2] improve sql querying of tags, fixes #221 * prepare statements in constructor and reuse them * make the class autoclosable * close resultsets after they are used * fix checkstyle warnings --- .../util/tagtranslator/TagTranslator.java | 130 +++++++++++------- 1 file changed, 82 insertions(+), 48 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java index 3eec3ab31..996c69a95 100644 --- a/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java @@ -4,6 +4,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.EnumSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -20,7 +21,7 @@ * Easily translate your textual tags and roles to OSHDB's internal * representation (encoded as integers) and vice versa. * - * This class handles missing/not-found data in the following ways: + *

This class handles missing/not-found data in the following ways:

* */ -public class TagTranslator { +public class TagTranslator implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(TagTranslator.class); + private final PreparedStatement keyIdQuery; + private final PreparedStatement keyTxtQuery; + private final PreparedStatement valueIdQuery; + private final PreparedStatement valueTxtQuery; + private final PreparedStatement roleIdQuery; + private final PreparedStatement roleTxtQuery; + private final Map keyToInt; private final Map keyToString; private final Map tagToInt; @@ -66,12 +74,44 @@ public TagTranslator(Connection conn) throws OSHDBKeytablesNotFoundException { EnumSet keyTables = EnumSet.of(TableNames.E_KEY, TableNames.E_KEYVALUE, TableNames.E_ROLE); for (TableNames table : keyTables) { - try { - this.conn.prepareStatement("select 1 from " + table.toString() + " LIMIT 1").execute(); + try (Statement testTablePresentQuery = this.conn.createStatement()) { + testTablePresentQuery.execute("select 1 from " + table.toString() + " LIMIT 1"); } catch (SQLException e) { throw new OSHDBKeytablesNotFoundException(); } } + + // create prepared statements for querying tags from keytables + try { + keyIdQuery = conn.prepareStatement( + "select ID from " + TableNames.E_KEY.toString() + " where KEY.TXT = ?;"); + keyTxtQuery = conn.prepareStatement( + "select TXT from " + TableNames.E_KEY.toString() + " where KEY.ID = ?;"); + valueIdQuery = conn.prepareStatement("select k.ID as KEYID,kv.VALUEID as VALUEID" + + " from " + TableNames.E_KEYVALUE.toString() + " kv" + + " inner join " + TableNames.E_KEY.toString() + " k on k.ID = kv.KEYID" + + " where k.TXT = ? and kv.TXT = ?;"); + valueTxtQuery = conn.prepareStatement("select k.TXT as KEYTXT,kv.TXT as VALUETXT" + + " from " + TableNames.E_KEYVALUE.toString() + " kv" + + " inner join " + TableNames.E_KEY.toString() + " k on k.ID = kv.KEYID" + + " where k.ID = ? and kv.VALUEID = ?;"); + roleIdQuery = conn + .prepareStatement("select ID from " + TableNames.E_ROLE.toString() + " where TXT = ?;"); + roleTxtQuery = conn + .prepareStatement("select TXT from " + TableNames.E_ROLE.toString() + " where ID = ?;"); + } catch (SQLException e) { + throw new OSHDBKeytablesNotFoundException(); + } + } + + @Override + public void close() throws Exception { + keyIdQuery.close(); + keyTxtQuery.close(); + valueIdQuery.close(); + valueTxtQuery.close(); + roleIdQuery.close(); + roleTxtQuery.close(); } /** @@ -95,12 +135,12 @@ public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { return this.keyToInt.get(key); } OSHDBTagKey keyInt; - try (PreparedStatement keystmt = this.conn.prepareStatement( - "select ID from " + TableNames.E_KEY.toString() + " where KEY.TXT = ?;")) { - keystmt.setString(1, key.toString()); - ResultSet keys = keystmt.executeQuery(); - keys.next(); - keyInt = new OSHDBTagKey(keys.getInt("ID")); + try { + keyIdQuery.setString(1, key.toString()); + try (ResultSet keys = keyIdQuery.executeQuery()) { + keys.next(); + keyInt = new OSHDBTagKey(keys.getInt("ID")); + } } catch (SQLException ex) { LOG.info("Unable to find tag key {} in keytables.", key); keyInt = new OSHDBTagKey(getFakeId(key.toString())); @@ -133,12 +173,12 @@ public OSMTagKey getOSMTagKeyOf(OSHDBTagKey key) { return this.keyToString.get(key); } OSMTagKey keyString; - try (PreparedStatement keystmt = - this.conn.prepareStatement("select TXT from KEY where KEY.ID = ?;")) { - keystmt.setInt(1, key.toInt()); - ResultSet keys = keystmt.executeQuery(); - keys.next(); - keyString = new OSMTagKey(keys.getString("TXT")); + try { + keyTxtQuery.setInt(1, key.toInt()); + try (ResultSet keys = keyTxtQuery.executeQuery()) { + keys.next(); + keyString = new OSMTagKey(keys.getString("TXT")); + } this.keyToInt.put(keyString, key); } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format( @@ -173,16 +213,13 @@ public OSHDBTag getOSHDBTagOf(OSMTag tag) { } OSHDBTag tagInt; // key or value is not in cache so let's go toInt them - try (PreparedStatement valstmt = - this.conn.prepareStatement("select k.ID as KEYID,kv.VALUEID as VALUEID " - + "from " + TableNames.E_KEYVALUE.toString() + " kv " - + "inner join " + TableNames.E_KEY.toString() + " k on k.ID = kv.KEYID " - + "WHERE k.TXT = ? and kv.TXT = ?;")) { - valstmt.setString(1, tag.getKey()); - valstmt.setString(2, tag.getValue()); - ResultSet values = valstmt.executeQuery(); - values.next(); - tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); + try { + valueIdQuery.setString(1, tag.getKey()); + valueIdQuery.setString(2, tag.getValue()); + try (ResultSet values = valueIdQuery.executeQuery()) { + values.next(); + tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); + } } catch (SQLException ex) { LOG.info("Unable to find tag {}={} in keytables.", tag.getKey(), tag.getValue()); tagInt = new OSHDBTag(this.getOSHDBTagKeyOf(tag.getKey()).toInt(), getFakeId(tag.getValue())); @@ -219,15 +256,13 @@ public OSMTag getOSMTagOf(OSHDBTag tag) { OSMTag tagString; // key or value is not in cache so let's go toInt them - try (PreparedStatement valstmt = - this.conn.prepareStatement("select k.TXT as KEYTXT,kv.TXT as VALUETXT from " - + TableNames.E_KEYVALUE.toString() + " kv inner join " + TableNames.E_KEY.toString() - + " k on k.ID = kv.KEYID WHERE k.ID = ? and kv.VALUEID = ?;")) { - valstmt.setInt(1, tag.getKey()); - valstmt.setInt(2, tag.getValue()); - ResultSet values = valstmt.executeQuery(); - values.next(); - tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); + try { + valueTxtQuery.setInt(1, tag.getKey()); + valueTxtQuery.setInt(2, tag.getValue()); + try (ResultSet values = valueTxtQuery.executeQuery()) { + values.next(); + tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); + } } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format( "Unable to find tag id %d=%d in keytables.", @@ -261,12 +296,12 @@ public OSHDBRole getOSHDBRoleOf(OSMRole role) { return this.roleToInt.get(role); } OSHDBRole roleInt; - try (PreparedStatement rolestmt = conn - .prepareStatement("select ID from " + TableNames.E_ROLE.toString() + " WHERE txt = ?;")) { - rolestmt.setString(1, role.toString()); - ResultSet roles = rolestmt.executeQuery(); - roles.next(); - roleInt = new OSHDBRole(roles.getInt("ID")); + try { + roleIdQuery.setString(1, role.toString()); + try (ResultSet roles = roleIdQuery.executeQuery()) { + roles.next(); + roleInt = new OSHDBRole(roles.getInt("ID")); + } } catch (SQLException ex) { LOG.info("Unable to find role {} in keytables.", role); roleInt = new OSHDBRole(getFakeId(role.toString())); @@ -299,13 +334,12 @@ public OSMRole getOSMRoleOf(OSHDBRole role) { return this.roleToString.get(role); } OSMRole roleString; - try (PreparedStatement Rolestmt = conn.prepareStatement( - "select TXT from " + TableNames.E_ROLE.toString() + " WHERE ID = ?;" - )) { - Rolestmt.setInt(1, role.toInt()); - ResultSet Roles = Rolestmt.executeQuery(); - Roles.next(); - roleString = new OSMRole(Roles.getString("TXT")); + try { + roleTxtQuery.setInt(1, role.toInt()); + try (ResultSet roles = roleTxtQuery.executeQuery()) { + roles.next(); + roleString = new OSMRole(roles.getString("TXT")); + } } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format( "Unable to find role id %d in keytables.", role.toInt() From cc149207f336b0a4442f30f6c03e727e37f572ad Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Mon, 6 Apr 2020 17:01:19 +0200 Subject: [PATCH 2/2] synchronize on prepared statement usage --- .../util/tagtranslator/TagTranslator.java | 66 +++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java index 996c69a95..3224bad75 100644 --- a/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/bigspatialdata/oshdb/util/tagtranslator/TagTranslator.java @@ -136,10 +136,12 @@ public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { } OSHDBTagKey keyInt; try { - keyIdQuery.setString(1, key.toString()); - try (ResultSet keys = keyIdQuery.executeQuery()) { - keys.next(); - keyInt = new OSHDBTagKey(keys.getInt("ID")); + synchronized (keyIdQuery) { + keyIdQuery.setString(1, key.toString()); + try (ResultSet keys = keyIdQuery.executeQuery()) { + keys.next(); + keyInt = new OSHDBTagKey(keys.getInt("ID")); + } } } catch (SQLException ex) { LOG.info("Unable to find tag key {} in keytables.", key); @@ -174,12 +176,14 @@ public OSMTagKey getOSMTagKeyOf(OSHDBTagKey key) { } OSMTagKey keyString; try { - keyTxtQuery.setInt(1, key.toInt()); - try (ResultSet keys = keyTxtQuery.executeQuery()) { - keys.next(); - keyString = new OSMTagKey(keys.getString("TXT")); + synchronized (keyTxtQuery) { + keyTxtQuery.setInt(1, key.toInt()); + try (ResultSet keys = keyTxtQuery.executeQuery()) { + keys.next(); + keyString = new OSMTagKey(keys.getString("TXT")); + } + this.keyToInt.put(keyString, key); } - this.keyToInt.put(keyString, key); } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format( "Unable to find tag key id %d in keytables.", key.toInt() @@ -214,11 +218,13 @@ public OSHDBTag getOSHDBTagOf(OSMTag tag) { OSHDBTag tagInt; // key or value is not in cache so let's go toInt them try { - valueIdQuery.setString(1, tag.getKey()); - valueIdQuery.setString(2, tag.getValue()); - try (ResultSet values = valueIdQuery.executeQuery()) { - values.next(); - tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); + synchronized (valueIdQuery) { + valueIdQuery.setString(1, tag.getKey()); + valueIdQuery.setString(2, tag.getValue()); + try (ResultSet values = valueIdQuery.executeQuery()) { + values.next(); + tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); + } } } catch (SQLException ex) { LOG.info("Unable to find tag {}={} in keytables.", tag.getKey(), tag.getValue()); @@ -257,11 +263,13 @@ public OSMTag getOSMTagOf(OSHDBTag tag) { // key or value is not in cache so let's go toInt them try { - valueTxtQuery.setInt(1, tag.getKey()); - valueTxtQuery.setInt(2, tag.getValue()); - try (ResultSet values = valueTxtQuery.executeQuery()) { - values.next(); - tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); + synchronized (valueTxtQuery) { + valueTxtQuery.setInt(1, tag.getKey()); + valueTxtQuery.setInt(2, tag.getValue()); + try (ResultSet values = valueTxtQuery.executeQuery()) { + values.next(); + tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); + } } } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format( @@ -297,10 +305,12 @@ public OSHDBRole getOSHDBRoleOf(OSMRole role) { } OSHDBRole roleInt; try { - roleIdQuery.setString(1, role.toString()); - try (ResultSet roles = roleIdQuery.executeQuery()) { - roles.next(); - roleInt = new OSHDBRole(roles.getInt("ID")); + synchronized (roleIdQuery) { + roleIdQuery.setString(1, role.toString()); + try (ResultSet roles = roleIdQuery.executeQuery()) { + roles.next(); + roleInt = new OSHDBRole(roles.getInt("ID")); + } } } catch (SQLException ex) { LOG.info("Unable to find role {} in keytables.", role); @@ -335,10 +345,12 @@ public OSMRole getOSMRoleOf(OSHDBRole role) { } OSMRole roleString; try { - roleTxtQuery.setInt(1, role.toInt()); - try (ResultSet roles = roleTxtQuery.executeQuery()) { - roles.next(); - roleString = new OSMRole(roles.getString("TXT")); + synchronized (roleTxtQuery) { + roleTxtQuery.setInt(1, role.toInt()); + try (ResultSet roles = roleTxtQuery.executeQuery()) { + roles.next(); + roleString = new OSMRole(roles.getString("TXT")); + } } } catch (SQLException ex) { throw new OSHDBTagOrRoleNotFoundException(String.format(