SNOW-1163203: Increased Max LOB size in metadata (#1806)

src/main/java/net/snowflake/client/core/ObjectMapperFactory.java

@@ -11,8 +11,9 @@
  */
 public class ObjectMapperFactory {
  @SnowflakeJdbcInternalApi
- // Snowflake allows up to 16M string size and returns base64 encoded value that makes it up to 23M
- public static final int DEFAULT_MAX_JSON_STRING_LEN = 23_000_000;
+ // Snowflake allows up to 128M (after updating Max LOB size) string size and returns base64
+ // encoded value that makes it up to 180M
+ public static final int DEFAULT_MAX_JSON_STRING_LEN = 180_000_000;
 
  @SnowflakeJdbcInternalApi
  public static final String MAX_JSON_STRING_LENGTH_JVM =

src/main/java/net/snowflake/client/jdbc/SnowflakeDatabaseMetaData.java

@@ -35,6 +35,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Pattern;
 import net.snowflake.client.core.ObjectMapperFactory;
@@ -140,6 +141,12 @@ public class SnowflakeDatabaseMetaData implements DatabaseMetaData {
  "VECTOR",
  "VIEW");
 
+ private static final String MAX_VARCHAR_BINARY_SIZE_PARAM_NAME =
+ "VARCHAR_AND_BINARY_MAX_SIZE_IN_RESULT";
+
+ // Defaults to 16MB
+ private static final int DEFAULT_MAX_LOB_SIZE = 16777216;
+
  private final Connection connection;
 
  private final SFBaseSession session;
@@ -911,14 +918,17 @@ public boolean supportsOpenStatementsAcrossRollback() throws SQLException {
  public int getMaxBinaryLiteralLength() throws SQLException {
  logger.trace("int getMaxBinaryLiteralLength()", false);
  raiseSQLExceptionIfConnectionIsClosed();
- return 8388608;
+ return getMaxCharLiteralLength() / 2; // hex instead of octal, thus divided by 2
  }
 
  @Override
  public int getMaxCharLiteralLength() throws SQLException {
  logger.trace("int getMaxCharLiteralLength()", false);
  raiseSQLExceptionIfConnectionIsClosed();
- return 16777216;
+ Optional<Integer> maxLiteralLengthFromSession =
+ Optional.ofNullable(
+ (Integer) session.getOtherParameter(MAX_VARCHAR_BINARY_SIZE_PARAM_NAME));
+ return maxLiteralLengthFromSession.orElse(DEFAULT_MAX_LOB_SIZE);
  }
 
  @Override
@@ -1348,9 +1358,9 @@ else if (i == 0) {
  typeName.substring(typeName.indexOf('(') + 1, typeName.indexOf(')')));
  nextRow[16] = char_octet_len;
  } else if (type == Types.CHAR || type == Types.VARCHAR) {
- nextRow[16] = 16777216;
+ nextRow[16] = getMaxCharLiteralLength();
  } else if (type == Types.BINARY || type == Types.VARBINARY) {
- nextRow[16] = 8388608;
+ nextRow[16] = getMaxBinaryLiteralLength();
  }
  } else {
  nextRow[16] = null;
@@ -3570,9 +3580,9 @@ public ResultSet getFunctionColumns(
  typeName.substring(typeName.indexOf('(') + 1, typeName.indexOf(')')));
  nextRow[13] = char_octet_len;
  } else if (type == Types.CHAR || type == Types.VARCHAR) {
- nextRow[13] = 16777216;
+ nextRow[13] = getMaxCharLiteralLength();
  } else if (type == Types.BINARY || type == Types.VARBINARY) {
- nextRow[13] = 8388608;
+ nextRow[13] = getMaxBinaryLiteralLength();
  }
  } else {
  nextRow[13] = null;

src/main/java/net/snowflake/client/jdbc/SnowflakeStatementV1.java

@@ -68,9 +68,6 @@ class SnowflakeStatementV1 implements Statement, SnowflakeStatement {
  // timeout in seconds
  private int queryTimeout = 0;
 
- // max field size limited to 16MB
- private final int maxFieldSize = 16777216;
-
  SFBaseStatement sfBaseStatement;
 
  private boolean poolable;
@@ -640,7 +637,7 @@ public ResultSet getGeneratedKeys() throws SQLException {
  public int getMaxFieldSize() throws SQLException {
  logger.trace("getMaxFieldSize()", false);
  raiseSQLExceptionIfStatementIsClosed();
- return maxFieldSize;
+ return connection.getMetaData().getMaxCharLiteralLength();
  }
 
  @Override

src/test/java/net/snowflake/client/jdbc/DatabaseMetaDataIT.java

@@ -59,6 +59,10 @@ public class DatabaseMetaDataIT extends BaseJDBCTest {
  + " $$\n"
  + " ;";
 
+ public static final int EXPECTED_MAX_CHAR_LENGTH = 16777216;
+
+ public static final int EXPECTED_MAX_BINARY_LENGTH = 8388608;
+
  @Test
  public void testGetConnection() throws SQLException {
  try (Connection connection = getConnection()) {
@@ -698,9 +702,9 @@ public void testDatabaseMetadata() throws SQLException {
  assertEquals("$", metaData.getExtraNameCharacters());
  assertEquals("\"", metaData.getIdentifierQuoteString());
  assertEquals(0, getSizeOfResultSet(metaData.getIndexInfo(null, null, null, true, true)));
- assertEquals(8388608, metaData.getMaxBinaryLiteralLength());
+ assertEquals(EXPECTED_MAX_BINARY_LENGTH, metaData.getMaxBinaryLiteralLength());
  assertEquals(255, metaData.getMaxCatalogNameLength());
- assertEquals(16777216, metaData.getMaxCharLiteralLength());
+ assertEquals(EXPECTED_MAX_CHAR_LENGTH, metaData.getMaxCharLiteralLength());
  assertEquals(255, metaData.getMaxColumnNameLength());
  assertEquals(0, metaData.getMaxColumnsInGroupBy());
  assertEquals(0, metaData.getMaxColumnsInIndex());

src/test/java/net/snowflake/client/jdbc/DatabaseMetaDataInternalIT.java

@@ -3,6 +3,8 @@
  */
 package net.snowflake.client.jdbc;
 
+import static net.snowflake.client.jdbc.DatabaseMetaDataIT.EXPECTED_MAX_BINARY_LENGTH;
+import static net.snowflake.client.jdbc.DatabaseMetaDataIT.verifyResultSetMetaDataColumns;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -59,7 +61,7 @@ static void initMetaData(Connection con) throws SQLException {
  st.execute("create or replace database JDBC_DB2");
  st.execute("create or replace schema JDBC_SCHEMA21");
  st.execute("create or replace table JDBC_TBL211(colA string)");
- st.execute("create or replace table JDBC_BIN(bin1 binary, bin2 binary(100))");
+ st.execute("create or replace table JDBC_BIN(bin1 binary(8388608), bin2 binary(100))");
 
  // st.execute("create or replace table JDBC_TBL211(colA string(25) NOT NULL DEFAULT
  // 'defstring')");
@@ -111,7 +113,7 @@ public void testGetColumn() throws SQLException {
 
  resultSet = databaseMetaData.getColumns(null, "JDBC_SCHEMA21", "JDBC_BIN", "BIN1");
  resultSet.next();
- assertEquals(8388608, resultSet.getInt("COLUMN_SIZE"));
+ assertEquals(EXPECTED_MAX_BINARY_LENGTH, resultSet.getInt("COLUMN_SIZE"));
  assertEquals(1, getSizeOfResultSet(resultSet) + 1);
 
  resultSet = databaseMetaData.getColumns(null, "JDBC_SCHEMA21", "JDBC_BIN", "BIN2");
@@ -187,8 +189,7 @@ public void testGetFunctions() throws SQLException {
 
  // test each column return the right value
  resultSet = databaseMetaData.getFunctions("JDBC_DB1", "JDBC_SCHEMA11", "JDBCFUNCTEST111");
- DatabaseMetaDataIT.verifyResultSetMetaDataColumns(
- resultSet, DBMetadataResultSetMetadata.GET_FUNCTIONS);
+ verifyResultSetMetaDataColumns(resultSet, DBMetadataResultSetMetadata.GET_FUNCTIONS);
  resultSet.next();
  assertEquals("JDBC_DB1", resultSet.getString("FUNCTION_CAT"));
  assertEquals("JDBC_SCHEMA11", resultSet.getString("FUNCTION_SCHEM"));

src/test/java/net/snowflake/client/jdbc/DatabaseMetaDataInternalLatestIT.java

@@ -91,7 +91,7 @@ public void testGetFunctionColumns() throws SQLException {
  + "sharedCol decimal)");
  statement.execute(
  "create or replace function JDBC_DB1.JDBC_SCHEMA11.FUNC112 "
- + "() RETURNS TABLE(colA string, colB decimal, bin2 binary, sharedCol decimal) COMMENT= 'returns "
+ + "() RETURNS TABLE(colA string(16777216), colB decimal, bin2 binary(8388608), sharedCol decimal) COMMENT= 'returns "
  + "table of 4 columns'"
  + " as 'select JDBC_DB1.JDBC_SCHEMA11.JDBC_TBL111.colA, JDBC_DB1.JDBC_SCHEMA11.JDBC_TBL111.colB, "
  + "JDBC_DB1.JDBC_SCHEMA11.BIN_TABLE.bin2, JDBC_DB1.JDBC_SCHEMA11.BIN_TABLE.sharedCol from JDBC_DB1"
@@ -173,7 +173,8 @@ public void testGetFunctionColumns() throws SQLException {
  assertEquals(10, resultSet.getInt("RADIX"));
  assertEquals(DatabaseMetaData.functionNullableUnknown, resultSet.getInt("NULLABLE"));
  assertEquals("returns table of 4 columns", resultSet.getString("REMARKS"));
- assertEquals(16777216, resultSet.getInt("CHAR_OCTET_LENGTH"));
+ assertEquals(
+ databaseMetaData.getMaxCharLiteralLength(), resultSet.getInt("CHAR_OCTET_LENGTH"));
  assertEquals(1, resultSet.getInt("ORDINAL_POSITION"));
  assertEquals("", resultSet.getString("IS_NULLABLE"));
  assertEquals(
@@ -213,7 +214,8 @@ public void testGetFunctionColumns() throws SQLException {
  assertEquals(10, resultSet.getInt("RADIX"));
  assertEquals(DatabaseMetaData.functionNullableUnknown, resultSet.getInt("NULLABLE"));
  assertEquals("returns table of 4 columns", resultSet.getString("REMARKS"));
- assertEquals(8388608, resultSet.getInt("CHAR_OCTET_LENGTH"));
+ assertEquals(
+ databaseMetaData.getMaxBinaryLiteralLength(), resultSet.getInt("CHAR_OCTET_LENGTH"));
  assertEquals(3, resultSet.getInt("ORDINAL_POSITION"));
  assertEquals("", resultSet.getString("IS_NULLABLE"));
  assertEquals(

src/test/java/net/snowflake/client/jdbc/DatabaseMetaDataLatestIT.java

@@ -3,6 +3,8 @@
  */
 package net.snowflake.client.jdbc;
 
+import static net.snowflake.client.jdbc.DatabaseMetaDataIT.EXPECTED_MAX_BINARY_LENGTH;
+import static net.snowflake.client.jdbc.DatabaseMetaDataIT.EXPECTED_MAX_CHAR_LENGTH;
 import static net.snowflake.client.jdbc.DatabaseMetaDataIT.verifyResultSetMetaDataColumns;
 import static net.snowflake.client.jdbc.SnowflakeDatabaseMetaData.NumericFunctionsSupported;
 import static net.snowflake.client.jdbc.SnowflakeDatabaseMetaData.StringFunctionsSupported;
@@ -772,8 +774,8 @@ public void testGetFunctionColumns() throws Exception {
  "create or replace table JDBC_TBL111(colA string, colB decimal, colC " + "timestamp)");
  /* Create a UDF that returns a table made up of 4 columns from 2 different tables, joined together */
  statement.execute(
- "create or replace function FUNC112 () RETURNS TABLE(colA string, colB decimal, bin2"
- + " binary, sharedCol decimal) COMMENT= 'returns table of 4 columns' as 'select"
+ "create or replace function FUNC112 () RETURNS TABLE(colA string(16777216), colB decimal, bin2 "
+ + "binary(8388608) , sharedCol decimal) COMMENT= 'returns table of 4 columns' as 'select"
  + " JDBC_TBL111.colA, JDBC_TBL111.colB, BIN_TABLE.bin2, BIN_TABLE.sharedCol from"
  + " JDBC_TBL111 inner join BIN_TABLE on JDBC_TBL111.colB =BIN_TABLE.sharedCol'");
  DatabaseMetaData metaData = connection.getMetaData();
@@ -877,7 +879,7 @@ public void testGetFunctionColumns() throws Exception {
  assertEquals(DatabaseMetaData.functionNullableUnknown, resultSet.getInt("NULLABLE"));
  assertEquals("returns table of 4 columns", resultSet.getString("REMARKS"));
  // char octet length column is not supported and always returns 0
- assertEquals(16777216, resultSet.getInt("CHAR_OCTET_LENGTH"));
+ assertEquals(EXPECTED_MAX_CHAR_LENGTH, resultSet.getInt("CHAR_OCTET_LENGTH"));
  assertEquals(1, resultSet.getInt("ORDINAL_POSITION"));
  // is_nullable column is not supported and always returns empty string
  assertEquals("", resultSet.getString("IS_NULLABLE"));
@@ -927,7 +929,7 @@ public void testGetFunctionColumns() throws Exception {
  assertEquals(DatabaseMetaData.functionNullableUnknown, resultSet.getInt("NULLABLE"));
  assertEquals("returns table of 4 columns", resultSet.getString("REMARKS"));
  // char octet length column is not supported and always returns 0
- assertEquals(8388608, resultSet.getInt("CHAR_OCTET_LENGTH"));
+ assertEquals(EXPECTED_MAX_BINARY_LENGTH, resultSet.getInt("CHAR_OCTET_LENGTH"));
  assertEquals(3, resultSet.getInt("ORDINAL_POSITION"));
  // is_nullable column is not supported and always returns empty string
  assertEquals("", resultSet.getString("IS_NULLABLE"));
@@ -1222,8 +1224,8 @@ public void testGetColumns() throws Throwable {
  statement.execute(
  "create or replace table "
  + targetTable
- + "(C1 int, C2 varchar(100), C3 string default '', C4 number(18,4), C5 double,"
- + " C6 boolean, C7 date not null, C8 time, C9 timestamp_ntz(7), C10 binary,C11"
+ + "(C1 int, C2 varchar(100), C3 string(16777216) default '', C4 number(18,4), C5 double,"
+ + " C6 boolean, C7 date not null, C8 time, C9 timestamp_ntz(7), C10 binary(8388608),C11"
  + " variant, C12 timestamp_ltz(8), C13 timestamp_tz(3))");
 
  DatabaseMetaData metaData = connection.getMetaData();
@@ -1290,14 +1292,14 @@ public void testGetColumns() throws Throwable {
  assertEquals("C3", resultSet.getString("COLUMN_NAME"));
  assertEquals(Types.VARCHAR, resultSet.getInt("DATA_TYPE"));
  assertEquals("VARCHAR", resultSet.getString("TYPE_NAME"));
- assertEquals(16777216, resultSet.getInt("COLUMN_SIZE"));
+ assertEquals(EXPECTED_MAX_CHAR_LENGTH, resultSet.getInt("COLUMN_SIZE"));
  assertEquals(0, resultSet.getInt("DECIMAL_DIGITS"));
  assertEquals(0, resultSet.getInt("NUM_PREC_RADIX"));
  assertEquals(ResultSetMetaData.columnNullable, resultSet.getInt("NULLABLE"));
  assertEquals("", resultSet.getString("REMARKS"));
  assertEquals("", resultSet.getString("COLUMN_DEF"));
 
- assertEquals(16777216, resultSet.getInt("CHAR_OCTET_LENGTH"));
+ assertEquals(EXPECTED_MAX_CHAR_LENGTH, resultSet.getInt("CHAR_OCTET_LENGTH"));
  assertEquals(3, resultSet.getInt("ORDINAL_POSITION"));
  assertEquals("YES", resultSet.getString("IS_NULLABLE"));
  assertNull(resultSet.getString("SCOPE_CATALOG"));
@@ -1465,7 +1467,7 @@ public void testGetColumns() throws Throwable {
  assertEquals("C10", resultSet.getString("COLUMN_NAME"));
  assertEquals(Types.BINARY, resultSet.getInt("DATA_TYPE"));
  assertEquals("BINARY", resultSet.getString("TYPE_NAME"));
- assertEquals(8388608, resultSet.getInt("COLUMN_SIZE"));
+ assertEquals(EXPECTED_MAX_BINARY_LENGTH, resultSet.getInt("COLUMN_SIZE"));
  assertEquals(0, resultSet.getInt("DECIMAL_DIGITS"));
  assertEquals(0, resultSet.getInt("NUM_PREC_RADIX"));
  assertEquals(ResultSetMetaData.columnNullable, resultSet.getInt("NULLABLE"));