Skip to content

Commit 58f82af

Browse files
Torch3333Torch3333
committed
For Oracle, change the data type for BLOB column to support storing up to 2GB
1 parent 6ad930f commit 58f82af

28 files changed

+806
-139
lines changed

‎core/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,7 @@ task integrationTestJdbc(type: Test) {
251251
options {
252252
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
253253
}
254+
maxHeapSize = "4g"
254255
}
255256

256257
task integrationTestMultiStorage(type: Test) {

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
418418

419419
@Override
420420
protected boolean isIndexOnBlobColumnSupported() {
421-
return !JdbcTestUtils.isDb2(rdbEngine);
421+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
422422
}
423423
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminCaseSensitivityIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
498498

499499
@Override
500500
protected boolean isIndexOnBlobColumnSupported() {
501-
return !JdbcTestUtils.isDb2(rdbEngine);
501+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
502502
}
503503
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ private LinkedHashMap<String, String> prepareColumnsForDb2() {
482482
columns.put("col17", "NCLOB(512)");
483483
columns.put("col18", "BINARY(5)");
484484
columns.put("col19", "VARBINARY(512)");
485-
columns.put("col20", "BLOB(1024)");
485+
columns.put("col20", "BLOB(2G)");
486486
columns.put("col21", "CHAR(5) FOR BIT DATA");
487487
columns.put("col22", "VARCHAR(512) FOR BIT DATA");
488488
columns.put("col23", "DATE");

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,6 @@ public void alterColumnType_Sqlite_AlterColumnType_ShouldThrowUnsupportedOperati
498498

499499
@Override
500500
protected boolean isIndexOnBlobColumnSupported() {
501-
return !JdbcTestUtils.isDb2(rdbEngine);
501+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
502502
}
503503
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseColumnValueIntegrationTest.java‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,41 @@
11
package com.scalar.db.storage.jdbc;
22

3+
import static org.assertj.core.api.Assertions.assertThat;
4+
35
import com.scalar.db.api.DistributedStorageColumnValueIntegrationTestBase;
6+
import com.scalar.db.api.Get;
7+
import com.scalar.db.api.Put;
8+
import com.scalar.db.api.PutBuilder;
9+
import com.scalar.db.api.Result;
10+
import com.scalar.db.api.TableMetadata;
411
import com.scalar.db.config.DatabaseConfig;
12+
import com.scalar.db.exception.storage.ExecutionException;
13+
import com.scalar.db.io.BigIntColumn;
14+
import com.scalar.db.io.BlobColumn;
15+
import com.scalar.db.io.BooleanColumn;
516
import com.scalar.db.io.Column;
617
import com.scalar.db.io.DataType;
18+
import com.scalar.db.io.DateColumn;
19+
import com.scalar.db.io.DoubleColumn;
20+
import com.scalar.db.io.FloatColumn;
21+
import com.scalar.db.io.IntColumn;
22+
import com.scalar.db.io.Key;
23+
import com.scalar.db.io.TextColumn;
24+
import com.scalar.db.io.TimeColumn;
25+
import com.scalar.db.io.TimestampColumn;
26+
import com.scalar.db.io.TimestampTZColumn;
727
import com.scalar.db.util.TestUtils;
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import java.util.Optional;
831
import java.util.Properties;
932
import java.util.Random;
33+
import java.util.stream.Stream;
34+
import org.junit.jupiter.api.Test;
35+
import org.junit.jupiter.api.condition.EnabledIf;
36+
import org.junit.jupiter.params.ParameterizedTest;
37+
import org.junit.jupiter.params.provider.Arguments;
38+
import org.junit.jupiter.params.provider.MethodSource;
1039

1140
public class JdbcDatabaseColumnValueIntegrationTest
1241
extends DistributedStorageColumnValueIntegrationTestBase {
@@ -68,4 +97,181 @@ protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType)
6897
}
6998
return super.getColumnWithMaxValue(columnName, dataType);
7099
}
100+
// TODO Test this for all storages
101+
@EnabledIf("isDb2OrOracle")
102+
@ParameterizedTest()
103+
@MethodSource("provideBlobSizes")
104+
public void put_largeBlobData_ShouldWorkCorrectly(int blobSize, String humanReadableBlobSize)
105+
throws ExecutionException {
106+
String tableName = TABLE + "_large_single_blob";
107+
try {
108+
// Arrange
109+
TableMetadata.Builder metadata =
110+
TableMetadata.newBuilder()
111+
.addColumn(COL_NAME1, DataType.INT)
112+
.addColumn(COL_NAME2, DataType.BLOB)
113+
.addPartitionKey(COL_NAME1);
114+
115+
admin.createTable(namespace, tableName, metadata.build(), true, getCreationOptions());
116+
byte[] blobData = createLargeBlob(blobSize);
117+
Put put =
118+
Put.newBuilder()
119+
.namespace(namespace)
120+
.table(tableName)
121+
.partitionKey(Key.ofInt(COL_NAME1, 1))
122+
.blobValue(COL_NAME2, blobData)
123+
.build();
124+
125+
// Act
126+
storage.put(put);
127+
128+
// Assert
129+
Optional<Result> optionalResult =
130+
storage.get(
131+
Get.newBuilder()
132+
.namespace(namespace)
133+
.table(tableName)
134+
.partitionKey(Key.ofInt(COL_NAME1, 1))
135+
.build());
136+
assertThat(optionalResult).isPresent();
137+
Result result = optionalResult.get();
138+
assertThat(result.getColumns().get(COL_NAME2).getBlobValueAsBytes()).isEqualTo(blobData);
139+
} finally {
140+
admin.dropTable(namespace, tableName, true);
141+
}
142+
}
143+
144+
Stream<Arguments> provideBlobSizes() {
145+
List<Arguments> args = new ArrayList<>();
146+
if (isOracle()) {
147+
// As explained in
148+
// `com.scalar.db.storage.jdbc.RdbEngineOracle.bindBlobColumnToPreparedStatement()`,
149+
// handing a BLOB size bigger than 32,767 bytes requires a workaround so we particularly test
150+
// values around it.
151+
args.add(Arguments.of(32_766, "32.766 KB"));
152+
args.add(Arguments.of(32_767, "32.767 KB"));
153+
}
154+
args.add(Arguments.of(100_000_000, "100 MB"));
155+
return args.stream();
156+
}
157+
158+
@EnabledIf("isOracle")
159+
@Test
160+
public void put_largeBlobData_WithMultipleBlobColumnsShouldWorkCorrectly()
161+
throws ExecutionException {
162+
String tableName = TABLE + "_large_multiples_blob";
163+
try {
164+
// Arrange
165+
TableMetadata.Builder metadata =
166+
TableMetadata.newBuilder()
167+
.addColumn(COL_NAME1, DataType.INT)
168+
.addColumn(COL_NAME2, DataType.BLOB)
169+
.addColumn(COL_NAME3, DataType.BLOB)
170+
.addPartitionKey(COL_NAME1);
171+
172+
admin.createTable(namespace, tableName, metadata.build(), true, getCreationOptions());
173+
byte[] blobDataCol2 = createLargeBlob(32_766);
174+
byte[] blobDataCol3 = createLargeBlob(5000);
175+
Put put =
176+
Put.newBuilder()
177+
.namespace(namespace)
178+
.table(tableName)
179+
.partitionKey(Key.ofInt(COL_NAME1, 1))
180+
.blobValue(COL_NAME2, blobDataCol2)
181+
.blobValue(COL_NAME3, blobDataCol3)
182+
.build();
183+
184+
// Act
185+
storage.put(put);
186+
187+
// Assert
188+
Optional<Result> optionalResult =
189+
storage.get(
190+
Get.newBuilder()
191+
.namespace(namespace)
192+
.table(tableName)
193+
.partitionKey(Key.ofInt(COL_NAME1, 1))
194+
.build());
195+
assertThat(optionalResult).isPresent();
196+
Result result = optionalResult.get();
197+
assertThat(result.getColumns().get(COL_NAME2).getBlobValueAsBytes()).isEqualTo(blobDataCol2);
198+
assertThat(result.getColumns().get(COL_NAME3).getBlobValueAsBytes()).isEqualTo(blobDataCol3);
199+
} finally {
200+
admin.dropTable(namespace, tableName, true);
201+
}
202+
}
203+
204+
@EnabledIf("isOracle")
205+
@Test
206+
public void put_largeBlobData_WithAllColumnsTypesShouldWorkCorrectly() throws ExecutionException {
207+
// Arrange
208+
IntColumn partitionKeyValue = (IntColumn) getColumnWithMaxValue(PARTITION_KEY, DataType.INT);
209+
BooleanColumn col1Value = (BooleanColumn) getColumnWithMaxValue(COL_NAME1, DataType.BOOLEAN);
210+
IntColumn col2Value = (IntColumn) getColumnWithMaxValue(COL_NAME2, DataType.INT);
211+
BigIntColumn col3Value = (BigIntColumn) getColumnWithMaxValue(COL_NAME3, DataType.BIGINT);
212+
FloatColumn col4Value = (FloatColumn) getColumnWithMaxValue(COL_NAME4, DataType.FLOAT);
213+
DoubleColumn col5Value = (DoubleColumn) getColumnWithMaxValue(COL_NAME5, DataType.DOUBLE);
214+
TextColumn col6Value = (TextColumn) getColumnWithMaxValue(COL_NAME6, DataType.TEXT);
215+
BlobColumn col7Value = BlobColumn.of(COL_NAME7, createLargeBlob(32_766));
216+
DateColumn col8Value = (DateColumn) getColumnWithMaxValue(COL_NAME8, DataType.DATE);
217+
TimeColumn col9Value = (TimeColumn) getColumnWithMaxValue(COL_NAME9, DataType.TIME);
218+
TimestampTZColumn col10Value =
219+
(TimestampTZColumn) getColumnWithMaxValue(COL_NAME10, DataType.TIMESTAMPTZ);
220+
TimestampColumn column11Value = null;
221+
if (isTimestampTypeSupported()) {
222+
column11Value = (TimestampColumn) getColumnWithMaxValue(COL_NAME11, DataType.TIMESTAMP);
223+
}
224+
225+
PutBuilder.Buildable put =
226+
Put.newBuilder()
227+
.namespace(namespace)
228+
.table(TABLE)
229+
.partitionKey(Key.newBuilder().add(partitionKeyValue).build())
230+
.value(col1Value)
231+
.value(col2Value)
232+
.value(col3Value)
233+
.value(col4Value)
234+
.value(col5Value)
235+
.value(col6Value)
236+
.value(col7Value)
237+
.value(col8Value)
238+
.value(col9Value)
239+
.value(col10Value);
240+
if (isTimestampTypeSupported()) {
241+
put.value(column11Value);
242+
}
243+
// Act
244+
storage.put(put.build());
245+
246+
// Assert
247+
assertResult(
248+
partitionKeyValue,
249+
col1Value,
250+
col2Value,
251+
col3Value,
252+
col4Value,
253+
col5Value,
254+
col6Value,
255+
col7Value,
256+
col8Value,
257+
col9Value,
258+
col10Value,
259+
column11Value);
260+
}
261+
262+
private byte[] createLargeBlob(int size) {
263+
byte[] blob = new byte[size];
264+
random.nextBytes(blob);
265+
return blob;
266+
}
267+
268+
@SuppressWarnings("unused")
269+
private boolean isDb2OrOracle() {
270+
return JdbcEnv.isOracle() || JdbcEnv.isDb2();
271+
}
272+
273+
@SuppressWarnings("unused")
274+
private boolean isOracle() {
275+
return JdbcEnv.isOracle();
276+
}
71277
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseConditionalMutationIntegrationTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,9 @@ protected Column<?> getColumnWithRandomValue(
4242
}
4343
return super.getColumnWithRandomValue(random, columnName, dataType);
4444
}
45+
46+
@Override
47+
protected boolean isConditionOnBlobColumnSupported() {
48+
return !JdbcTestUtils.isOracle(rdbEngine);
49+
}
4550
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseCrossPartitionScanIntegrationTest.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ protected Stream<Arguments> provideColumnsForCNFConditionsTest() {
8383

8484
@Override
8585
protected boolean isOrderingOnBlobColumnSupported() {
86-
return !JdbcTestUtils.isDb2(rdbEngine);
86+
return !(JdbcTestUtils.isDb2(rdbEngine) || JdbcTestUtils.isOracle(rdbEngine));
87+
}
88+
89+
@Override
90+
protected boolean isConditionOnBlobColumnSupported() {
91+
return !JdbcTestUtils.isOracle(rdbEngine);
8792
}
8893
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultipleClusteringKeyScanIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ protected List<DataType> getDataTypes() {
100100
rdbEngine,
101101
ImmutableMap.of(
102102
RdbEngineOracle.class,
103-
ImmutableList.of(DataType.TIMESTAMPTZ),
103+
ImmutableList.of(DataType.TIMESTAMPTZ, DataType.BLOB),
104104
RdbEngineDb2.class,
105105
ImmutableList.of(DataType.BLOB)));
106106
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseMultiplePartitionKeyIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ protected List<DataType> getDataTypes() {
9494
rdbEngine,
9595
ImmutableMap.of(
9696
RdbEngineOracle.class,
97-
ImmutableList.of(DataType.TIMESTAMPTZ),
97+
ImmutableList.of(DataType.TIMESTAMPTZ, DataType.BLOB),
9898
RdbEngineYugabyte.class,
9999
ImmutableList.of(DataType.FLOAT, DataType.DOUBLE),
100100
RdbEngineDb2.class,

0 commit comments

Comments
 (0)