Backport to branch(3) : For IBM Db2, change the data type for BLOB column to support storing up to 2GB

core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java

@@ -122,8 +122,8 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
   }
 
   @Override
-  protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
-    // "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
+  protected boolean isCreateIndexOnTextColumnEnabled() {
+    // "admin.createIndex()" for TEXT column fails (the "create index" query runs
     // indefinitely) on the Db2 community edition docker version which we use for the CI.
     // However, the index creation is successful on Db2 hosted on IBM Cloud.
     // So we disable these tests until the issue with the Db2 community edition is resolved.
@@ -180,4 +180,9 @@ public void renameColumn_Db2_ForPrimaryOrIndexKeyColumn_ShouldThrowUnsupportedOp
       admin.dropTable(namespace1, TABLE4, true);
     }
   }
+
+  @Override
+  protected boolean isIndexOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminCaseSensitivityIntegrationTest.java

@@ -106,8 +106,8 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
   }
 
   @Override
-  protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
-    // "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
+  protected boolean isCreateIndexOnTextColumnEnabled() {
+    // "admin.createIndex()" for TEXT column fails (the "create index" query runs
     // indefinitely) on Db2 community edition version but works on Db2 hosted on IBM Cloud.
     // So we disable these tests until the issue is resolved.
     return !JdbcTestUtils.isDb2(rdbEngine);
@@ -172,4 +172,9 @@ public void renameColumn_Db2_ForPrimaryOrIndexKeyColumn_ShouldThrowUnsupportedOp
       admin.dropTable(getNamespace1(), getTable4(), true);
     }
   }
+
+  @Override
+  protected boolean isIndexOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java

@@ -105,8 +105,8 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
   }
 
   @Override
-  protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
-    // "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
+  protected boolean isCreateIndexOnTextColumnEnabled() {
+    // "admin.createIndex()" for TEXT columns fails (the "create index" query runs
     // indefinitely) on Db2 community edition version but works on Db2 hosted on IBM Cloud.
     // So we disable these tests until the issue is resolved.
     return !JdbcTestUtils.isDb2(rdbEngine);
@@ -171,4 +171,9 @@ public void renameColumn_Db2_ForPrimaryOrIndexKeyColumn_ShouldThrowUnsupportedOp
       admin.dropTable(getNamespace1(), getTable4(), true);
     }
   }
+
+  @Override
+  protected boolean isIndexOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseCrossPartitionScanIntegrationTest.java

@@ -80,4 +80,9 @@ protected Stream<Arguments> provideColumnsForCNFConditionsTest() {
     }
     return Stream.of(Arguments.of(allColumnNames));
   }
+
+  @Override
+  protected boolean isOrderingOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultipleClusteringKeyScanIntegrationTest.java

@@ -94,9 +94,14 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
   @Override
   protected List<DataType> getDataTypes() {
     // TIMESTAMP WITH TIME ZONE type cannot be used as a primary key in Oracle
+    // BLOB type cannot be used as a clustering key in Db2
     return JdbcTestUtils.filterDataTypes(
         super.getDataTypes(),
         rdbEngine,
-        ImmutableMap.of(RdbEngineOracle.class, ImmutableList.of(DataType.TIMESTAMPTZ)));
+        ImmutableMap.of(
+            RdbEngineOracle.class,
+            ImmutableList.of(DataType.TIMESTAMPTZ),
+            RdbEngineDb2.class,
+            ImmutableList.of(DataType.BLOB)));
   }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultiplePartitionKeyIntegrationTest.java

@@ -88,13 +88,16 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
   protected List<DataType> getDataTypes() {
     // TIMESTAMP WITH TIME ZONE type cannot be used as a primary key in Oracle
     // FLOAT and DOUBLE types cannot be used as partition key in Yugabyte
+    // BLOB type cannot be used as a partition key in Db2
     return JdbcTestUtils.filterDataTypes(
         super.getDataTypes(),
         rdbEngine,
         ImmutableMap.of(
             RdbEngineOracle.class,
             ImmutableList.of(DataType.TIMESTAMPTZ),
             RdbEngineYugabyte.class,
-            ImmutableList.of(DataType.FLOAT, DataType.DOUBLE)));
+            ImmutableList.of(DataType.FLOAT, DataType.DOUBLE),
+            RdbEngineDb2.class,
+            ImmutableList.of(DataType.BLOB)));
   }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseSecondaryIndexIntegrationTest.java

@@ -1,12 +1,17 @@
 package com.scalar.db.storage.jdbc;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
 import com.scalar.db.api.DistributedStorageSecondaryIndexIntegrationTestBase;
 import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.io.Column;
 import com.scalar.db.io.DataType;
 import com.scalar.db.util.TestUtils;
+import java.util.Arrays;
 import java.util.Properties;
 import java.util.Random;
+import java.util.Set;
 
 public class JdbcDatabaseSecondaryIndexIntegrationTest
     extends DistributedStorageSecondaryIndexIntegrationTestBase {
@@ -68,4 +73,14 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
     }
     return super.getColumnWithMaxValue(columnName, dataType);
   }
+
+  @Override
+  protected Set<DataType> getSecondaryIndexTypes() {
+    // BLOB type cannot be used as a secondary index in Db2
+    return Sets.newHashSet(
+        JdbcTestUtils.filterDataTypes(
+            Arrays.asList(DataType.values()),
+            rdbEngine,
+            ImmutableMap.of(RdbEngineDb2.class, ImmutableList.of(DataType.BLOB))));
+  }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseSingleClusteringKeyScanIntegrationTest.java

@@ -70,9 +70,14 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
   @Override
   protected List<DataType> getClusteringKeyTypes() {
     // TIMESTAMP WITH TIME ZONE type cannot be used as a primary key in Oracle
+    // BLOB type cannot be used as a clustering key in Db2
     return JdbcTestUtils.filterDataTypes(
         super.getClusteringKeyTypes(),
         rdbEngine,
-        ImmutableMap.of(RdbEngineOracle.class, ImmutableList.of(DataType.TIMESTAMPTZ)));
+        ImmutableMap.of(
+            RdbEngineOracle.class,
+            ImmutableList.of(DataType.TIMESTAMPTZ),
+            RdbEngineDb2.class,
+            ImmutableList.of(DataType.BLOB)));
   }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseSinglePartitionKeyIntegrationTest.java

@@ -71,13 +71,16 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
   protected List<DataType> getPartitionKeyTypes() {
     // TIMESTAMP WITH TIME ZONE type cannot be used as a primary key in Oracle
     // FLOAT and DOUBLE types cannot be used as partition key in Yugabyte
+    // BLOB type cannot be used as a partition key in Db2
     return JdbcTestUtils.filterDataTypes(
         super.getPartitionKeyTypes(),
         rdbEngine,
         ImmutableMap.of(
             RdbEngineOracle.class,
             ImmutableList.of(DataType.TIMESTAMPTZ),
             RdbEngineYugabyte.class,
-            ImmutableList.of(DataType.FLOAT, DataType.DOUBLE)));
+            ImmutableList.of(DataType.FLOAT, DataType.DOUBLE),
+            RdbEngineDb2.class,
+            ImmutableList.of(DataType.BLOB)));
   }
 }

core/src/integration-test/java/com/scalar/db/storage/jdbc/SingleCrudOperationTransactionAdminIntegrationTestWithJdbcDatabase.java

@@ -106,8 +106,8 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
   }
 
   @Override
-  protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
-    // "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
+  protected boolean isCreateIndexOnTextColumnEnabled() {
+    // "admin.createIndex()" for TEXT column fails (the "create index" query runs
     // indefinitely) on the Db2 community edition docker version which we use for the CI.
     // However, the index creation is successful on Db2 hosted on IBM Cloud.
     // So we disable these tests until the issue with the Db2 community edition is resolved.
@@ -164,4 +164,9 @@ public void renameColumn_Db2_ForPrimaryOrIndexKeyColumn_ShouldThrowUnsupportedOp
       admin.dropTable(namespace1, TABLE4, true);
     }
   }
+
+  @Override
+  protected boolean isIndexOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminIntegrationTest.java

@@ -173,8 +173,8 @@ public void dropCoordinatorTables_IfExist_CoordinatorTablesDoNotExist_ShouldNotT
   }
 
   @Override
-  protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
-    // "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
+  protected boolean isCreateIndexOnTextColumnEnabled() {
+    // "admin.createIndex()" for TEXT column fails (the "create index" query runs
     // indefinitely) on the Db2 community edition docker version which we use for the CI.
     // However, the index creation is successful on Db2 hosted on IBM Cloud.
     // So we disable these tests until the issue with the Db2 community edition is resolved.
@@ -231,4 +231,9 @@ public void renameColumn_Db2_ForPrimaryOrIndexKeyColumn_ShouldThrowUnsupportedOp
       admin.dropTable(namespace1, TABLE4, true);
     }
   }
+
+  @Override
+  protected boolean isIndexOnBlobColumnSupported() {
+    return !JdbcTestUtils.isDb2(rdbEngine);
+  }
 }

core/src/main/java/com/scalar/db/common/CoreError.java

@@ -742,6 +742,18 @@ public enum CoreError implements ScalarDbError {
       "The TIMESTAMP type is not supported in Cassandra. Column: %s",
       "",
       ""),
+  JDBC_DB2_INDEX_OR_KEY_ON_BLOB_COLUMN_NOT_SUPPORTED(
+      Category.USER_ERROR,
+      "0228",
+      "With Db2, using a BLOB column as partition key, clustering key or secondary index is not supported.",
+      "",
+      ""),
+  JDBC_DB2_CROSS_PARTITION_SCAN_ORDERING_ON_BLOB_COLUMN_NOT_SUPPORTED(
+      Category.USER_ERROR,
+      "0229",
+      "With Db2, setting an ordering on a BLOB column when using a cross partition scan operation is not supported. Ordering: %s",
+      "",
+      ""),
 
   //
   // Errors for the concurrency error category

core/src/main/java/com/scalar/db/common/checker/OperationChecker.java

@@ -159,7 +159,7 @@ private void check(ScanAll scanAll) throws ExecutionException {
       throw new IllegalArgumentException(
           CoreError.OPERATION_CHECK_ERROR_CROSS_PARTITION_SCAN_ORDERING.buildMessage(scanAll));
     }
-    checkOrderings(scanAll, metadata);
+    checkOrderingsForScanAll(scanAll, metadata);
 
     if (!config.isCrossPartitionScanFilteringEnabled() && !scanAll.getConjunctions().isEmpty()) {
       throw new IllegalArgumentException(
@@ -258,7 +258,7 @@ private void checkOrderings(Scan scan, TableMetadata metadata) {
     }
   }
 
-  private void checkOrderings(ScanAll scanAll, TableMetadata metadata) {
+  protected void checkOrderingsForScanAll(ScanAll scanAll, TableMetadata metadata) {
     for (Scan.Ordering ordering : scanAll.getOrderings()) {
       if (!metadata.getColumnNames().contains(ordering.getColumnName())) {
         throw new IllegalArgumentException(

core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java

@@ -59,8 +59,8 @@ public JdbcDatabase(DatabaseConfig databaseConfig) {
     TableMetadataManager tableMetadataManager =
         new TableMetadataManager(jdbcAdmin, databaseConfig.getMetadataCacheExpirationTimeSecs());
     OperationChecker operationChecker =
-        new OperationChecker(
-            databaseConfig, tableMetadataManager, new StorageInfoProvider(jdbcAdmin));
+        new JdbcOperationChecker(
+            databaseConfig, tableMetadataManager, new StorageInfoProvider(jdbcAdmin), rdbEngine);
 
     jdbcService =
         new JdbcService(

core/src/main/java/com/scalar/db/storage/jdbc/JdbcOperationChecker.java

@@ -0,0 +1,29 @@
+package com.scalar.db.storage.jdbc;
+
+import com.scalar.db.api.ScanAll;
+import com.scalar.db.api.TableMetadata;
+import com.scalar.db.common.StorageInfoProvider;
+import com.scalar.db.common.TableMetadataManager;
+import com.scalar.db.common.checker.OperationChecker;
+import com.scalar.db.config.DatabaseConfig;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public class JdbcOperationChecker extends OperationChecker {
+  private final RdbEngineStrategy rdbEngine;
+
+  @SuppressFBWarnings("EI_EXPOSE_REP2")
+  public JdbcOperationChecker(
+      DatabaseConfig config,
+      TableMetadataManager tableMetadataManager,
+      StorageInfoProvider storageInfoProvider,
+      RdbEngineStrategy rdbEngine) {
+    super(config, tableMetadataManager, storageInfoProvider);
+    this.rdbEngine = rdbEngine;
+  }
+
+  @Override
+  protected void checkOrderingsForScanAll(ScanAll scanAll, TableMetadata metadata) {
+    super.checkOrderingsForScanAll(scanAll, metadata);
+    rdbEngine.throwIfCrossPartitionScanOrderingOnBlobColumnNotSupported(scanAll, metadata);
+  }
+}

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineDb2.java

@@ -6,6 +6,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.ibm.db2.jcc.DB2BaseDataSource;
 import com.scalar.db.api.LikeExpression;
+import com.scalar.db.api.Scan.Ordering;
+import com.scalar.db.api.ScanAll;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.common.CoreError;
 import com.scalar.db.exception.storage.ExecutionException;
@@ -31,6 +33,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
@@ -64,7 +67,7 @@ public String getDataTypeForEngine(DataType scalarDbDataType) {
       case BIGINT:
         return "BIGINT";
       case BLOB:
-        return "VARBINARY(32672)";
+        return "BLOB(2G)";
       case BOOLEAN:
         return "BOOLEAN";
       case FLOAT:
@@ -352,7 +355,8 @@ public String getDataTypeForKey(DataType dataType) {
       case TEXT:
         return "VARCHAR(" + keyColumnSize + ") NOT NULL";
       case BLOB:
-        return "VARBINARY(" + keyColumnSize + ") NOT NULL";
+        throw new UnsupportedOperationException(
+            CoreError.JDBC_DB2_INDEX_OR_KEY_ON_BLOB_COLUMN_NOT_SUPPORTED.buildMessage());
       default:
         return getDataTypeForEngine(dataType) + " NOT NULL";
     }
@@ -365,7 +369,8 @@ public String getDataTypeForSecondaryIndex(DataType dataType) {
       case TEXT:
         return "VARCHAR(" + keyColumnSize + ")";
       case BLOB:
-        return "VARBINARY(" + keyColumnSize + ")";
+        throw new UnsupportedOperationException(
+            CoreError.JDBC_DB2_INDEX_OR_KEY_ON_BLOB_COLUMN_NOT_SUPPORTED.buildMessage());
       default:
         return null;
     }
@@ -518,4 +523,19 @@ private String getProjection(String columnName, DataType dataType) {
     }
     return enclose(columnName);
   }
+
+  @Override
+  public void throwIfCrossPartitionScanOrderingOnBlobColumnNotSupported(
+      ScanAll scanAll, TableMetadata metadata) {
+    Optional<Ordering> orderingOnBlobColumn =
+        scanAll.getOrderings().stream()
+            .filter(
+                ordering -> metadata.getColumnDataType(ordering.getColumnName()) == DataType.BLOB)
+            .findFirst();
+    if (orderingOnBlobColumn.isPresent()) {
+      throw new UnsupportedOperationException(
+          CoreError.JDBC_DB2_CROSS_PARTITION_SCAN_ORDERING_ON_BLOB_COLUMN_NOT_SUPPORTED
+              .buildMessage(orderingOnBlobColumn.get()));
+    }
+  }
 }

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineStrategy.java

@@ -1,6 +1,7 @@
 package com.scalar.db.storage.jdbc;
 
 import com.scalar.db.api.LikeExpression;
+import com.scalar.db.api.ScanAll;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.exception.storage.ExecutionException;
 import com.scalar.db.io.DataType;
@@ -283,4 +284,16 @@ default void setConnectionToReadOnly(Connection connection, boolean readOnly)
       throws SQLException {
     connection.setReadOnly(readOnly);
   }
+
+  /**
+   * Throws an exception if a cross-partition scan operation with ordering on a blob column is
+   * specified and is not supported in the underlying storage.
+   *
+   * @param scanAll the ScanAll operation
+   * @param metadata the table metadata
+   * @throws UnsupportedOperationException if the ScanAll operation contains an ordering on a blob
+   *     column, and it is not supported in the underlying storage
+   */
+  default void throwIfCrossPartitionScanOrderingOnBlobColumnNotSupported(
+      ScanAll scanAll, TableMetadata metadata) {}
 }