From 13c50fa650ef2f963333d6fcfded6bfec8d7fc65 Mon Sep 17 00:00:00 2001 From: Rich Loveland Date: Tue, 28 May 2019 16:22:09 -0400 Subject: [PATCH] Simplify JDBC code sample Summary of changes: - Update 'Build a Java App with CockroachDB' to use: - the Java DAO pattern - JDBC - code for automatically retrying in case of txn retry errors - an example of fast bulk insertion using JDBC batching - Add a 'Recommended Practices' section that includes information about IMPORT, recommended batch size, and the JDBC INSERT rewriting flag Does all of the above for versions 19.1 and 19.2. Fixes #4621, #3578, #4399. --- _includes/v19.1/app/BasicExample.java | 440 ++++++++++++++++++ _includes/v19.1/app/BasicSample.java | 55 --- .../v19.1/app/insecure/BasicExample.java | 436 +++++++++++++++++ _includes/v19.1/app/insecure/BasicSample.java | 51 -- _includes/v19.2/app/BasicExample.java | 440 ++++++++++++++++++ .../v19.2/app/insecure/BasicExample.java | 436 +++++++++++++++++ v19.1/build-a-java-app-with-cockroachdb.md | 276 +++++------ v19.1/savepoint.md | 2 +- v19.1/transactions.md | 4 +- v19.2/build-a-java-app-with-cockroachdb.md | 278 +++++------ v19.2/savepoint.md | 2 +- v19.2/transactions.md | 4 +- 12 files changed, 2049 insertions(+), 375 deletions(-) create mode 100644 _includes/v19.1/app/BasicExample.java delete mode 100644 _includes/v19.1/app/BasicSample.java create mode 100644 _includes/v19.1/app/insecure/BasicExample.java delete mode 100644 _includes/v19.1/app/insecure/BasicSample.java create mode 100644 _includes/v19.2/app/BasicExample.java create mode 100644 _includes/v19.2/app/insecure/BasicExample.java diff --git a/_includes/v19.1/app/BasicExample.java b/_includes/v19.1/app/BasicExample.java new file mode 100644 index 00000000000..88a391c5e12 --- /dev/null +++ b/_includes/v19.1/app/BasicExample.java @@ -0,0 +1,440 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setSsl(true); + ds.setSslMode("require"); + ds.setSslCert("certs/client.maxroach.crt"); + ds.setSslKey("certs/client.maxroach.key.pk8"); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic, maybe (if FORCE_RETRY is + // true). This method is only used to test the retry logic. + // It is not necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in account ID 1? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. Note that this calls an internal + CockroachDB function that can only be run by the 'root' user, + and will fail with an insufficient privileges error if you try + to run it as user 'maxroach'. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/_includes/v19.1/app/BasicSample.java b/_includes/v19.1/app/BasicSample.java deleted file mode 100644 index 25d326dd4e0..00000000000 --- a/_includes/v19.1/app/BasicSample.java +++ /dev/null @@ -1,55 +0,0 @@ -import java.sql.*; -import java.util.Properties; - -/* - Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. - - Then, compile and run this example like so: - - $ export CLASSPATH=.:/path/to/postgresql.jar - $ javac BasicSample.java && java BasicSample -*/ - -public class BasicSample { - public static void main(String[] args) - throws ClassNotFoundException, SQLException { - - // Load the Postgres JDBC driver. - Class.forName("org.postgresql.Driver"); - - // Connect to the "bank" database. - Properties props = new Properties(); - props.setProperty("user", "maxroach"); - props.setProperty("sslmode", "require"); - props.setProperty("sslrootcert", "certs/ca.crt"); - props.setProperty("sslkey", "certs/client.maxroach.key.pk8"); - props.setProperty("sslcert", "certs/client.maxroach.crt"); - props.setProperty("ApplicationName", "roachtest"); - - Connection db = DriverManager - .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); - - try { - // Create the "accounts" table. - db.createStatement() - .execute("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); - - // Insert two rows into the "accounts" table. - db.createStatement() - .execute("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); - - // Print out the balances. - System.out.println("Initial balances:"); - ResultSet res = db.createStatement() - .executeQuery("SELECT id, balance FROM accounts"); - while (res.next()) { - System.out.printf("\taccount %s: %s\n", - res.getInt("id"), - res.getInt("balance")); - } - } finally { - // Close the database connection. - db.close(); - } - } -} diff --git a/_includes/v19.1/app/insecure/BasicExample.java b/_includes/v19.1/app/insecure/BasicExample.java new file mode 100644 index 00000000000..c7e4f62fc09 --- /dev/null +++ b/_includes/v19.1/app/insecure/BasicExample.java @@ -0,0 +1,436 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic, maybe (if FORCE_RETRY is + // true). This method is only used to test the retry logic. + // It is not necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in account ID 1? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. Note that this calls an internal + CockroachDB function that can only be run by the 'root' user, + and will fail with an insufficient privileges error if you try + to run it as user 'maxroach'. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/_includes/v19.1/app/insecure/BasicSample.java b/_includes/v19.1/app/insecure/BasicSample.java deleted file mode 100644 index 001d38feb48..00000000000 --- a/_includes/v19.1/app/insecure/BasicSample.java +++ /dev/null @@ -1,51 +0,0 @@ -import java.sql.*; -import java.util.Properties; - -/* - Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. - - Then, compile and run this example like so: - - $ export CLASSPATH=.:/path/to/postgresql.jar - $ javac BasicSample.java && java BasicSample -*/ - -public class BasicSample { - public static void main(String[] args) - throws ClassNotFoundException, SQLException { - - // Load the Postgres JDBC driver. - Class.forName("org.postgresql.Driver"); - - // Connect to the "bank" database. - Properties props = new Properties(); - props.setProperty("user", "maxroach"); - props.setProperty("sslmode", "disable"); - - Connection db = DriverManager - .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); - - try { - // Create the "accounts" table. - db.createStatement() - .execute("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); - - // Insert two rows into the "accounts" table. - db.createStatement() - .execute("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); - - // Print out the balances. - System.out.println("Initial balances:"); - ResultSet res = db.createStatement() - .executeQuery("SELECT id, balance FROM accounts"); - while (res.next()) { - System.out.printf("\taccount %s: %s\n", - res.getInt("id"), - res.getInt("balance")); - } - } finally { - // Close the database connection. - db.close(); - } - } -} diff --git a/_includes/v19.2/app/BasicExample.java b/_includes/v19.2/app/BasicExample.java new file mode 100644 index 00000000000..88a391c5e12 --- /dev/null +++ b/_includes/v19.2/app/BasicExample.java @@ -0,0 +1,440 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setSsl(true); + ds.setSslMode("require"); + ds.setSslCert("certs/client.maxroach.crt"); + ds.setSslKey("certs/client.maxroach.key.pk8"); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic, maybe (if FORCE_RETRY is + // true). This method is only used to test the retry logic. + // It is not necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in account ID 1? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. Note that this calls an internal + CockroachDB function that can only be run by the 'root' user, + and will fail with an insufficient privileges error if you try + to run it as user 'maxroach'. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/_includes/v19.2/app/insecure/BasicExample.java b/_includes/v19.2/app/insecure/BasicExample.java new file mode 100644 index 00000000000..c7e4f62fc09 --- /dev/null +++ b/_includes/v19.2/app/insecure/BasicExample.java @@ -0,0 +1,436 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic, maybe (if FORCE_RETRY is + // true). This method is only used to test the retry logic. + // It is not necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in account ID 1? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. Note that this calls an internal + CockroachDB function that can only be run by the 'root' user, + and will fail with an insufficient privileges error if you try + to run it as user 'maxroach'. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/v19.1/build-a-java-app-with-cockroachdb.md b/v19.1/build-a-java-app-with-cockroachdb.md index bfb3100b58b..43bf4e2beb7 100644 --- a/v19.1/build-a-java-app-with-cockroachdb.md +++ b/v19.1/build-a-java-app-with-cockroachdb.md @@ -10,7 +10,7 @@ twitter: false -This tutorial shows you how build a simple Java application with CockroachDB using a PostgreSQL-compatible driver or ORM. +This tutorial shows you how to build a simple Java application with CockroachDB using a PostgreSQL-compatible driver or ORM. We have tested the [Java JDBC driver](https://jdbc.postgresql.org/) and the [Hibernate ORM](http://hibernate.org/) enough to claim **beta-level** support, so those are featured here. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. @@ -34,7 +34,7 @@ Download and set up the Java JDBC driver as described in the [official documenta ## Step 3. Generate a certificate for the `maxroach` user -Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. New in v19.1: You can pass the [`--also-generate-pkcs8-key` flag](create-security-certificates.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. @@ -45,105 +45,45 @@ $ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-direc ## Step 4. Run the Java code -Now that you have created a database and set up encryption keys, in this section you will: -- [Create a table and insert some rows](#basic1) -- [Execute a batch of statements as a transaction](#txn1) +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: - +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). -### Basic example +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. -First, use the following code to connect as the `maxroach` user and execute some basic SQL statements: create a table, insert rows, and read and print the rows. +| Step | Method | +|------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------| +| 1. Create an `accounts` table in the `bank` database | `BasicExampleDAO.createAccounts()` | +| 2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend | `BasicExampleDAO.updateAccounts(Map balance)` | +| 3. Transfer money from one account to another, printing out account balances before and after the transfer | `BasicExampleDAO.transferFunds(int from, int to, int amount)` | +| 4. Insert random account data using JDBC's bulk insertion support | `BasicExampleDAO.bulkInsertRandomAccountData()` | +| 5. Print out some account data | `BasicExampleDAO.readAccounts(int limit)` | +| 6. Drop the `accounts` table and perform any other necessary cleanup | `BasicExampleDAO.tearDown()` (This cleanup step means you can run this program more than once.) | -To run it: - -1. Download [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicSample.java), or create the file yourself and copy the code below. -2. Download [the PostgreSQL JDBC driver](https://jdbc.postgresql.org/download.html). -3. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): - - {% include copy-clipboard.html %} - ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar BasicSample.java - ~~~ - - {% include copy-clipboard.html %} - ~~~ shell - $ java -classpath .:/path/to/postgresql.jar BasicSample - ~~~ - - The output should be: - - ~~~ - Initial balances: - account 1: 1000 - account 2: 250 - ~~~ - -The contents of [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicSample.java): - -{% include copy-clipboard.html %} -~~~ java -{% include {{page.version.version}}/app/BasicSample.java %} -~~~ - - - -### Transaction example (with retry logic) - -Next, use the following code to execute a batch of statements as a [transaction](transactions.html) to transfer funds from one account to another. +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. To run it: -1. Download TxnSample.java, or create the file yourself and copy the code below. Note the use of [`SQLException.getSQLState()`](https://docs.oracle.com/javase/tutorial/jdbc/basics/sqlexception.html) instead of `getErrorCode()`. -2. Compile and run the code (again adding the PostgreSQL JDBC driver to your classpath): +1. Download [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java), or create the file yourself and copy the code below. +2. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): {% include copy-clipboard.html %} ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar TxnSample.java + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java ~~~ {% include copy-clipboard.html %} ~~~ shell - $ java -classpath .:/path/to/postgresql.jar TxnSample - ~~~ - - The output should be: - - ~~~ - account 1: 900 - account 2: 350 + $ java -classpath .:/path/to/postgresql.jar BasicExample ~~~ -{% include v19.1/client-transaction-retry.md %} +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java): {% include copy-clipboard.html %} ~~~ java -{% include {{page.version.version}}/app/TxnSample.java %} -~~~ - -To verify that funds were transferred from one account to another, start the [built-in SQL client](use-the-built-in-sql-client.html): - -{% include copy-clipboard.html %} -~~~ shell -$ cockroach sql --certs-dir=certs --database=bank -~~~ - -To check the account balances, issue the following statement: - -{% include copy-clipboard.html %} -~~~ sql -> SELECT id, balance FROM accounts; -~~~ - -~~~ -+----+---------+ -| id | balance | -+----+---------+ -| 1 | 900 | -| 2 | 350 | -+----+---------+ -(2 rows) +{% include {{page.version.version}}/app/BasicExample.java %} ~~~ @@ -156,93 +96,167 @@ To check the account balances, issue the following statement: ## Step 3. Run the Java code -Now that you have created a database, in this section you will: +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: -- [Create a table and insert some rows](#basic2) -- [Execute a batch of statements as a transaction](#txn2) +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). - +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. -### Basic example +1. Create an `accounts` table in the `bank` database (`BasicExampleDAO.createAccounts()`). +2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend (`BasicExampleDAO.updateAccounts(Map balance)`). +3. Transfer money from one account to another, printing out account balances before and after the transfer (`BasicExampleDAO.transferFunds(int from, int to, int amount)`). +4. Insert random account data using JDBC's bulk insertion support (`BasicExampleDAO.bulkInsertRandomAccountData()`). +5. Print out (some) account data (`BasicExampleDAO.readAccounts(int limit)`). +6. Drop the `accounts` table and perform any other necessary cleanup (`BasicExampleDAO.tearDown()`). (Note that you can run this program as many times as you want due to this cleanup step.) -First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. To run it: -1. Download [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicSample.java), or create the file yourself and copy the code below. +1. Download [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicExample.java), or create the file yourself and copy the code below. 2. Download [the PostgreSQL JDBC driver](https://jdbc.postgresql.org/download.html). 3. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): {% include copy-clipboard.html %} ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar BasicSample.java + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java ~~~ {% include copy-clipboard.html %} ~~~ shell - $ java -classpath .:/path/to/postgresql.jar BasicSample + $ java -classpath .:/path/to/postgresql.jar BasicExample ~~~ -The contents of [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicSample.java): +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicExample.java): {% include copy-clipboard.html %} ~~~ java -{% include {{page.version.version}}/app/insecure/BasicSample.java %} +{% include {{page.version.version}}/app/insecure/BasicExample.java %} ~~~ - + -### Transaction example (with retry logic) +The output will look like the following: -Next, use the following code to execute a batch of statements as a [transaction](transactions.html) to transfer funds from one account to another. +~~~ +BasicExampleDAO.createAccounts: + 'CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (1, 1000)' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (2, 250)' +BasicExampleDAO.updateAccounts: + => 2 total updated accounts +main: + => Account balances at time '11:54:06.904': + ID 1 => $1000 + ID 2 => $250 + +BasicExampleDAO.transferFunds: + 'UPSERT INTO accounts (id, balance) VALUES(1, ((SELECT balance FROM accounts WHERE id = 1) - 100)),(2, ((SELECT balance FROM accounts WHERE id = 2) + 100))' +BasicExampleDAO.transferFunds: + => $100 transferred between accounts 1 and 2, 2 rows updated +main: + => Account balances at time '11:54:06.985': + ID 1 => $900 + ID 2 => $350 + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (354685257, 158423397)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (206179866, 950590234)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (708995411, 892928833)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (500817884, 189050420)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + => finished, 512 total rows inserted + +BasicExampleDAO.readAccounts: + 'SELECT id, balance FROM accounts LIMIT 10' + id => 1 + balance => 900 + id => 2 + balance => 350 + id => 190756 + balance => 966414958 + id => 1002343 + balance => 243354081 + id => 1159751 + balance => 59745201 + id => 2193125 + balance => 346719279 + id => 2659707 + balance => 770266587 + id => 6819325 + balance => 511618834 + id => 9985390 + balance => 905049643 + id => 12256472 + balance => 913034434 + +BasicExampleDAO.tearDown: + 'DROP TABLE accounts' +~~~ -To run it: +## Recommended Practices -1. Download TxnSample.java, or create the file yourself and copy the code below. Note the use of [`SQLException.getSQLState()`](https://docs.oracle.com/javase/tutorial/jdbc/basics/sqlexception.html) instead of `getErrorCode()`. -2. Compile and run the code (again adding the PostgreSQL JDBC driver to your classpath): +### Use `IMPORT` to read in large data sets - {% include copy-clipboard.html %} - ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar TxnSample.java - ~~~ +If you are trying to get a large data set into CockroachDB all at once (a bulk import), avoid writing client-side code altogether and use the [`IMPORT`](import.html) statement instead. It is much faster and more efficient than making a series of [`INSERT`s](insert.html) and [`UPDATE`s](update.html). It bypasses the [SQL layer](architecture/sql-layer.html) altogether and writes directly to the [storage layer](architecture/storage-layer.html) of the database. - {% include copy-clipboard.html %} - ~~~ shell - $ java -classpath .:/path/to/postgresql.jar TxnSample - ~~~ +For more information about importing data from Postgres, see [Migrate from Postgres](migrate-from-postgres.html). -{% include v19.1/client-transaction-retry.md %} +For more information about importing data from MySQL, see [Migrate from MySQL](migrate-from-mysql.html). -{% include copy-clipboard.html %} -~~~ java -{% include {{page.version.version}}/app/insecure/TxnSample.java %} -~~~ +### Use `rewriteBatchedInserts` for increased speed -To verify that funds were transferred from one account to another, start the [built-in SQL client](use-the-built-in-sql-client.html): +We strongly recommend setting `rewriteBatchedInserts=true`; we have seen 2-3x performance improvements with it enabled. From [the JDBC connection parameters documentation](https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters): -{% include copy-clipboard.html %} -~~~ shell -$ cockroach sql --insecure --database=bank -~~~ +> This will change batch inserts from `insert into foo (col1, col2, col3) values (1,2,3)` into `insert into foo (col1, col2, col3) values (1,2,3), (4,5,6)` this provides 2-3x performance improvement -To check the account balances, issue the following statement: +### Use a batch size of 128 -{% include copy-clipboard.html %} -~~~ sql -> SELECT id, balance FROM accounts; -~~~ +PGJDBC's batching support only works with [powers of two](https://github.com/pgjdbc/pgjdbc/blob/7b52b0c9e5b9aa9a9c655bb68f23bf4ec57fd51c/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java#L1597), and will split batches of other sizes up into multiple sub-batches. This means that a batch of size 128 can be 6x faster than a batch of size 250. -~~~ -+----+---------+ -| id | balance | -+----+---------+ -| 1 | 900 | -| 2 | 350 | -+----+---------+ -(2 rows) -~~~ +The code snippet below shows a pattern for using a batch size of 128, and is taken from the longer example above (specifically, the `BasicExampleDAO.bulkInsertRandomAccountData()` method). - +Specifically, it does the following: + +1. Turn off auto-commit so you can manage the transaction lifecycle and thus the size of the batch inserts. +2. Given an overall update size of 500 rows (for example), split it into batches of size 128 and execute each batch in turn. +3. Finally, commit the batches of statements you've just executed. + +~~~ java +int BATCH_SIZE = 128; +connection.setAutoCommit(false); + +try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); // Verifying 128 rows in the batch + } + connection.commit(); +} +~~~ ## What's next? diff --git a/v19.1/savepoint.md b/v19.1/savepoint.md index e2d00d1235b..b2f5fe4be51 100644 --- a/v19.1/savepoint.md +++ b/v19.1/savepoint.md @@ -71,5 +71,5 @@ Applications using `SAVEPOINT` must also include functions to execute retries wi - [`ROLLBACK`](rollback-transaction.html) - [`BEGIN`](begin-transaction.html) - [`COMMIT`](commit-transaction.html) -- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic) +- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html) - [CockroachDB Architecture: Transaction Layer](architecture/transaction-layer.html) diff --git a/v19.1/transactions.md b/v19.1/transactions.md index cf7c47f7034..8ac71435aac 100644 --- a/v19.1/transactions.md +++ b/v19.1/transactions.md @@ -178,7 +178,7 @@ Implementing client-side retries requires using the following statements: For examples showing how to use these statements, see the following: - The [Syntax](#syntax) section of this page. -- Many of our [Build an App with CockroachDB](build-an-app-with-cockroachdb.html) tutorials show code samples for issuing retries. For an example showing how to implement the retry logic, see [the Java/JDBC tutorial](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic). +- Many of our [Build an App with CockroachDB](build-an-app-with-cockroachdb.html) tutorials show code samples for issuing retries. For an example showing how to implement the retry logic, see [the Java/JDBC tutorial](build-a-java-app-with-cockroachdb.html). ##### Client library support @@ -270,5 +270,5 @@ For more information about the relationship between these levels, see [this pape - [`SAVEPOINT`](savepoint.html) - [`RELEASE SAVEPOINT`](release-savepoint.html) - [`SHOW`](show-vars.html) -- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic) +- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html) - [CockroachDB Architecture: Transaction Layer](architecture/transaction-layer.html) diff --git a/v19.2/build-a-java-app-with-cockroachdb.md b/v19.2/build-a-java-app-with-cockroachdb.md index daa6db0c5c4..43bf4e2beb7 100644 --- a/v19.2/build-a-java-app-with-cockroachdb.md +++ b/v19.2/build-a-java-app-with-cockroachdb.md @@ -10,7 +10,7 @@ twitter: false -This tutorial shows you how build a simple Java application with CockroachDB using a PostgreSQL-compatible driver or ORM. +This tutorial shows you how to build a simple Java application with CockroachDB using a PostgreSQL-compatible driver or ORM. We have tested the [Java JDBC driver](https://jdbc.postgresql.org/) and the [Hibernate ORM](http://hibernate.org/) enough to claim **beta-level** support, so those are featured here. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. @@ -34,9 +34,9 @@ Download and set up the Java JDBC driver as described in the [official documenta ## Step 3. Generate a certificate for the `maxroach` user -Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. -You can pass the [`--also-generate-pkcs8-key` flag](create-security-certificates.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. +New in v19.1: You can pass the [`--also-generate-pkcs8-key` flag](create-security-certificates.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. {% include copy-clipboard.html %} ~~~ shell @@ -45,105 +45,45 @@ $ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-direc ## Step 4. Run the Java code -Now that you have created a database and set up encryption keys, in this section you will: -- [Create a table and insert some rows](#basic1) -- [Execute a batch of statements as a transaction](#txn1) +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: - +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). -### Basic example +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. -First, use the following code to connect as the `maxroach` user and execute some basic SQL statements: create a table, insert rows, and read and print the rows. +| Step | Method | +|------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------| +| 1. Create an `accounts` table in the `bank` database | `BasicExampleDAO.createAccounts()` | +| 2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend | `BasicExampleDAO.updateAccounts(Map balance)` | +| 3. Transfer money from one account to another, printing out account balances before and after the transfer | `BasicExampleDAO.transferFunds(int from, int to, int amount)` | +| 4. Insert random account data using JDBC's bulk insertion support | `BasicExampleDAO.bulkInsertRandomAccountData()` | +| 5. Print out some account data | `BasicExampleDAO.readAccounts(int limit)` | +| 6. Drop the `accounts` table and perform any other necessary cleanup | `BasicExampleDAO.tearDown()` (This cleanup step means you can run this program more than once.) | -To run it: - -1. Download [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/BasicSample.java), or create the file yourself and copy the code below. -2. Download [the PostgreSQL JDBC driver](https://jdbc.postgresql.org/download.html). -3. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): - - {% include copy-clipboard.html %} - ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar BasicSample.java - ~~~ - - {% include copy-clipboard.html %} - ~~~ shell - $ java -classpath .:/path/to/postgresql.jar BasicSample - ~~~ - - The output should be: - - ~~~ - Initial balances: - account 1: 1000 - account 2: 250 - ~~~ - -The contents of [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/BasicSample.java): - -{% include copy-clipboard.html %} -~~~ java -{% include {{page.version.version}}/app/BasicSample.java %} -~~~ - - - -### Transaction example (with retry logic) - -Next, use the following code to execute a batch of statements as a [transaction](transactions.html) to transfer funds from one account to another. +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. To run it: -1. Download TxnSample.java, or create the file yourself and copy the code below. Note the use of [`SQLException.getSQLState()`](https://docs.oracle.com/javase/tutorial/jdbc/basics/sqlexception.html) instead of `getErrorCode()`. -2. Compile and run the code (again adding the PostgreSQL JDBC driver to your classpath): +1. Download [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java), or create the file yourself and copy the code below. +2. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): {% include copy-clipboard.html %} ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar TxnSample.java + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java ~~~ {% include copy-clipboard.html %} ~~~ shell - $ java -classpath .:/path/to/postgresql.jar TxnSample - ~~~ - - The output should be: - - ~~~ - account 1: 900 - account 2: 350 + $ java -classpath .:/path/to/postgresql.jar BasicExample ~~~ -{% include {{ page.version.version }}/client-transaction-retry.md %} +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java): {% include copy-clipboard.html %} ~~~ java -{% include {{page.version.version}}/app/TxnSample.java %} -~~~ - -To verify that funds were transferred from one account to another, start the [built-in SQL client](use-the-built-in-sql-client.html): - -{% include copy-clipboard.html %} -~~~ shell -$ cockroach sql --certs-dir=certs --database=bank -~~~ - -To check the account balances, issue the following statement: - -{% include copy-clipboard.html %} -~~~ sql -> SELECT id, balance FROM accounts; -~~~ - -~~~ -+----+---------+ -| id | balance | -+----+---------+ -| 1 | 900 | -| 2 | 350 | -+----+---------+ -(2 rows) +{% include {{page.version.version}}/app/BasicExample.java %} ~~~ @@ -156,93 +96,167 @@ To check the account balances, issue the following statement: ## Step 3. Run the Java code -Now that you have created a database, in this section you will: +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: -- [Create a table and insert some rows](#basic2) -- [Execute a batch of statements as a transaction](#txn2) +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). - +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. -### Basic example +1. Create an `accounts` table in the `bank` database (`BasicExampleDAO.createAccounts()`). +2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend (`BasicExampleDAO.updateAccounts(Map balance)`). +3. Transfer money from one account to another, printing out account balances before and after the transfer (`BasicExampleDAO.transferFunds(int from, int to, int amount)`). +4. Insert random account data using JDBC's bulk insertion support (`BasicExampleDAO.bulkInsertRandomAccountData()`). +5. Print out (some) account data (`BasicExampleDAO.readAccounts(int limit)`). +6. Drop the `accounts` table and perform any other necessary cleanup (`BasicExampleDAO.tearDown()`). (Note that you can run this program as many times as you want due to this cleanup step.) -First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. To run it: -1. Download [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/insecure/BasicSample.java), or create the file yourself and copy the code below. +1. Download [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicExample.java), or create the file yourself and copy the code below. 2. Download [the PostgreSQL JDBC driver](https://jdbc.postgresql.org/download.html). 3. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): {% include copy-clipboard.html %} ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar BasicSample.java + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java ~~~ {% include copy-clipboard.html %} ~~~ shell - $ java -classpath .:/path/to/postgresql.jar BasicSample + $ java -classpath .:/path/to/postgresql.jar BasicExample ~~~ -The contents of [`BasicSample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/insecure/BasicSample.java): +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicExample.java): {% include copy-clipboard.html %} ~~~ java -{% include {{page.version.version}}/app/insecure/BasicSample.java %} +{% include {{page.version.version}}/app/insecure/BasicExample.java %} ~~~ - + -### Transaction example (with retry logic) +The output will look like the following: -Next, use the following code to execute a batch of statements as a [transaction](transactions.html) to transfer funds from one account to another. +~~~ +BasicExampleDAO.createAccounts: + 'CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (1, 1000)' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (2, 250)' +BasicExampleDAO.updateAccounts: + => 2 total updated accounts +main: + => Account balances at time '11:54:06.904': + ID 1 => $1000 + ID 2 => $250 + +BasicExampleDAO.transferFunds: + 'UPSERT INTO accounts (id, balance) VALUES(1, ((SELECT balance FROM accounts WHERE id = 1) - 100)),(2, ((SELECT balance FROM accounts WHERE id = 2) + 100))' +BasicExampleDAO.transferFunds: + => $100 transferred between accounts 1 and 2, 2 rows updated +main: + => Account balances at time '11:54:06.985': + ID 1 => $900 + ID 2 => $350 + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (354685257, 158423397)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (206179866, 950590234)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (708995411, 892928833)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (500817884, 189050420)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + => finished, 512 total rows inserted + +BasicExampleDAO.readAccounts: + 'SELECT id, balance FROM accounts LIMIT 10' + id => 1 + balance => 900 + id => 2 + balance => 350 + id => 190756 + balance => 966414958 + id => 1002343 + balance => 243354081 + id => 1159751 + balance => 59745201 + id => 2193125 + balance => 346719279 + id => 2659707 + balance => 770266587 + id => 6819325 + balance => 511618834 + id => 9985390 + balance => 905049643 + id => 12256472 + balance => 913034434 + +BasicExampleDAO.tearDown: + 'DROP TABLE accounts' +~~~ -To run it: +## Recommended Practices -1. Download TxnSample.java, or create the file yourself and copy the code below. Note the use of [`SQLException.getSQLState()`](https://docs.oracle.com/javase/tutorial/jdbc/basics/sqlexception.html) instead of `getErrorCode()`. -2. Compile and run the code (again adding the PostgreSQL JDBC driver to your classpath): +### Use `IMPORT` to read in large data sets - {% include copy-clipboard.html %} - ~~~ shell - $ javac -classpath .:/path/to/postgresql.jar TxnSample.java - ~~~ +If you are trying to get a large data set into CockroachDB all at once (a bulk import), avoid writing client-side code altogether and use the [`IMPORT`](import.html) statement instead. It is much faster and more efficient than making a series of [`INSERT`s](insert.html) and [`UPDATE`s](update.html). It bypasses the [SQL layer](architecture/sql-layer.html) altogether and writes directly to the [storage layer](architecture/storage-layer.html) of the database. - {% include copy-clipboard.html %} - ~~~ shell - $ java -classpath .:/path/to/postgresql.jar TxnSample - ~~~ +For more information about importing data from Postgres, see [Migrate from Postgres](migrate-from-postgres.html). -{% include {{ page.version.version }}/client-transaction-retry.md %} +For more information about importing data from MySQL, see [Migrate from MySQL](migrate-from-mysql.html). -{% include copy-clipboard.html %} -~~~ java -{% include {{page.version.version}}/app/insecure/TxnSample.java %} -~~~ +### Use `rewriteBatchedInserts` for increased speed -To verify that funds were transferred from one account to another, start the [built-in SQL client](use-the-built-in-sql-client.html): +We strongly recommend setting `rewriteBatchedInserts=true`; we have seen 2-3x performance improvements with it enabled. From [the JDBC connection parameters documentation](https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters): -{% include copy-clipboard.html %} -~~~ shell -$ cockroach sql --insecure --database=bank -~~~ +> This will change batch inserts from `insert into foo (col1, col2, col3) values (1,2,3)` into `insert into foo (col1, col2, col3) values (1,2,3), (4,5,6)` this provides 2-3x performance improvement -To check the account balances, issue the following statement: +### Use a batch size of 128 -{% include copy-clipboard.html %} -~~~ sql -> SELECT id, balance FROM accounts; -~~~ +PGJDBC's batching support only works with [powers of two](https://github.com/pgjdbc/pgjdbc/blob/7b52b0c9e5b9aa9a9c655bb68f23bf4ec57fd51c/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java#L1597), and will split batches of other sizes up into multiple sub-batches. This means that a batch of size 128 can be 6x faster than a batch of size 250. -~~~ -+----+---------+ -| id | balance | -+----+---------+ -| 1 | 900 | -| 2 | 350 | -+----+---------+ -(2 rows) -~~~ +The code snippet below shows a pattern for using a batch size of 128, and is taken from the longer example above (specifically, the `BasicExampleDAO.bulkInsertRandomAccountData()` method). - +Specifically, it does the following: + +1. Turn off auto-commit so you can manage the transaction lifecycle and thus the size of the batch inserts. +2. Given an overall update size of 500 rows (for example), split it into batches of size 128 and execute each batch in turn. +3. Finally, commit the batches of statements you've just executed. + +~~~ java +int BATCH_SIZE = 128; +connection.setAutoCommit(false); + +try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); // Verifying 128 rows in the batch + } + connection.commit(); +} +~~~ ## What's next? diff --git a/v19.2/savepoint.md b/v19.2/savepoint.md index e2d00d1235b..b2f5fe4be51 100644 --- a/v19.2/savepoint.md +++ b/v19.2/savepoint.md @@ -71,5 +71,5 @@ Applications using `SAVEPOINT` must also include functions to execute retries wi - [`ROLLBACK`](rollback-transaction.html) - [`BEGIN`](begin-transaction.html) - [`COMMIT`](commit-transaction.html) -- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic) +- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html) - [CockroachDB Architecture: Transaction Layer](architecture/transaction-layer.html) diff --git a/v19.2/transactions.md b/v19.2/transactions.md index cf7c47f7034..8ac71435aac 100644 --- a/v19.2/transactions.md +++ b/v19.2/transactions.md @@ -178,7 +178,7 @@ Implementing client-side retries requires using the following statements: For examples showing how to use these statements, see the following: - The [Syntax](#syntax) section of this page. -- Many of our [Build an App with CockroachDB](build-an-app-with-cockroachdb.html) tutorials show code samples for issuing retries. For an example showing how to implement the retry logic, see [the Java/JDBC tutorial](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic). +- Many of our [Build an App with CockroachDB](build-an-app-with-cockroachdb.html) tutorials show code samples for issuing retries. For an example showing how to implement the retry logic, see [the Java/JDBC tutorial](build-a-java-app-with-cockroachdb.html). ##### Client library support @@ -270,5 +270,5 @@ For more information about the relationship between these levels, see [this pape - [`SAVEPOINT`](savepoint.html) - [`RELEASE SAVEPOINT`](release-savepoint.html) - [`SHOW`](show-vars.html) -- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html#transaction-example-with-retry-logic) +- [Retryable transaction example code in Java using JDBC](build-a-java-app-with-cockroachdb.html) - [CockroachDB Architecture: Transaction Layer](architecture/transaction-layer.html)