Skip to content

Commit cb36f1b

Browse files
Torch3333Torch3333
committed
Add Db2 adapter
1 parent f7f0be9 commit cb36f1b

File tree

41 files changed

+1788
-124
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1788
-124
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1578,6 +1578,160 @@ jobs:
15781578
name: yugabytedb_2_integration_test_reports_${{ matrix.mode.label }}
15791579
path: core/build/reports/tests/integrationTestJdbc
15801580

1581+
integration-test-for-jdbc-db2-12-1:
1582+
name: Db2 12.1 integration test (${{ matrix.mode.label }})
1583+
runs-on: ubuntu-latest
1584+
1585+
services:
1586+
db2:
1587+
image: icr.io/db2_community/db2:12.1.1.0
1588+
env:
1589+
DB2INSTANCE: db2inst1
1590+
DB2INST1_PASSWORD: db2inst1
1591+
DBNAME: test_db
1592+
LICENSE: accept
1593+
ports:
1594+
- 50000:50000
1595+
options: --privileged --name db2
1596+
1597+
strategy:
1598+
matrix:
1599+
mode:
1600+
- label: default
1601+
group_commit_enabled: false
1602+
- label: with_group_commit
1603+
group_commit_enabled: true
1604+
1605+
steps:
1606+
- uses: actions/checkout@v4
1607+
1608+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
1609+
uses: actions/setup-java@v4
1610+
with:
1611+
java-version: ${{ env.JAVA_VERSION }}
1612+
distribution: ${{ env.JAVA_VENDOR }}
1613+
1614+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
1615+
uses: actions/setup-java@v4
1616+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
1617+
with:
1618+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
1619+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
1620+
1621+
- name: Login to Oracle container registry
1622+
uses: docker/login-action@v3
1623+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1624+
with:
1625+
registry: container-registry.oracle.com
1626+
username: ${{ secrets.OCR_USERNAME }}
1627+
password: ${{ secrets.OCR_TOKEN }}
1628+
1629+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
1630+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1631+
run: |
1632+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
1633+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
1634+
1635+
- name: Setup Gradle
1636+
uses: gradle/actions/setup-gradle@v4
1637+
1638+
- name: Wait for the container to be ready
1639+
timeout-minutes: 10
1640+
run : |
1641+
while ! docker logs db2 2>&1 | grep -q "Setup has completed"
1642+
do
1643+
echo "Container is not yet ready"
1644+
sleep 5s
1645+
done
1646+
echo "Container is ready"
1647+
1648+
- name: Execute Gradle 'integrationTestJdbc' task
1649+
run: ./gradlew integrationTestJdbc -Dscalardb.jdbc.url="jdbc:db2://localhost:50000/test_db" -Dscalardb.jdbc.username=db2inst1 -Dscalardb.jdbc.password=db2inst1 ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
1650+
1651+
- name: Upload Gradle test reports
1652+
if: always()
1653+
uses: actions/upload-artifact@v4
1654+
with:
1655+
name: db2_12.1_integration_test_reports_${{ matrix.mode.label }}
1656+
path: core/build/reports/tests/integrationTestJdbc
1657+
1658+
integration-test-for-jdbc-db2-11-5:
1659+
name: Db2 11.5 integration test (${{ matrix.mode.label }})
1660+
runs-on: ubuntu-latest
1661+
1662+
services:
1663+
db2:
1664+
image: icr.io/db2_community/db2:11.5.9.0
1665+
env:
1666+
DB2INSTANCE: db2inst1
1667+
DB2INST1_PASSWORD: db2inst1
1668+
DBNAME: test_db
1669+
LICENSE: accept
1670+
ports:
1671+
- 50000:50000
1672+
options: --privileged --name db2
1673+
1674+
strategy:
1675+
matrix:
1676+
mode:
1677+
- label: default
1678+
group_commit_enabled: false
1679+
- label: with_group_commit
1680+
group_commit_enabled: true
1681+
1682+
steps:
1683+
- uses: actions/checkout@v4
1684+
1685+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
1686+
uses: actions/setup-java@v4
1687+
with:
1688+
java-version: ${{ env.JAVA_VERSION }}
1689+
distribution: ${{ env.JAVA_VENDOR }}
1690+
1691+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
1692+
uses: actions/setup-java@v4
1693+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
1694+
with:
1695+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
1696+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
1697+
1698+
- name: Login to Oracle container registry
1699+
uses: docker/login-action@v3
1700+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1701+
with:
1702+
registry: container-registry.oracle.com
1703+
username: ${{ secrets.OCR_USERNAME }}
1704+
password: ${{ secrets.OCR_TOKEN }}
1705+
1706+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
1707+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1708+
run: |
1709+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
1710+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
1711+
1712+
- name: Setup Gradle
1713+
uses: gradle/actions/setup-gradle@v4
1714+
1715+
- name: Wait for the container to be ready
1716+
timeout-minutes: 10
1717+
run: |
1718+
while ! docker logs db2 2>&1 | grep -q "Setup has completed"
1719+
do
1720+
echo "Container is not yet ready"
1721+
sleep 5s
1722+
done
1723+
echo "Container is ready"
1724+
1725+
- name: Execute Gradle 'integrationTestJdbc' task
1726+
run: ./gradlew integrationTestJdbc -Dscalardb.jdbc.url="jdbc:db2://localhost:50000/test_db" -Dscalardb.jdbc.username=db2inst1 -Dscalardb.jdbc.password=db2inst1 ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
1727+
1728+
- name: Upload Gradle test reports
1729+
if: always()
1730+
uses: actions/upload-artifact@v4
1731+
with:
1732+
name: db2_11.5_integration_test_reports_${{ matrix.mode.label }}
1733+
path: core/build/reports/tests/integrationTestJdbc
1734+
15811735
integration-test-for-multi-storage:
15821736
name: Multi-storage integration test (${{ matrix.mode.label }})
15831737
runs-on: ubuntu-latest

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ subprojects {
3535
sqlserverDriverVersion = '12.8.1.jre8'
3636
sqliteDriverVersion = '3.49.1.0'
3737
yugabyteDriverVersion = '42.7.3-yb-4'
38+
db2DriverVersion= '12.1.0.0'
3839
mariadDbDriverVersion = '3.5.3'
3940
picocliVersion = '4.7.7'
4041
commonsTextVersion = '1.13.1'

‎core/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ dependencies {
107107
implementation "com.microsoft.sqlserver:mssql-jdbc:${sqlserverDriverVersion}"
108108
implementation "org.xerial:sqlite-jdbc:${sqliteDriverVersion}"
109109
implementation "com.yugabyte:jdbc-yugabytedb:${yugabyteDriverVersion}"
110+
implementation "com.ibm.db2:jcc:${db2DriverVersion}"
110111
implementation ("org.mariadb.jdbc:mariadb-java-client:${mariadDbDriverVersion}") {
111112
exclude group: 'org.slf4j', module: 'slf4j-api'
112113
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,32 @@
11
package com.scalar.db.storage.jdbc;
22

3+
import com.scalar.db.config.DatabaseConfig;
34
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
45
import com.scalar.db.util.AdminTestUtils;
56
import java.util.Properties;
67

78
public class ConsensusCommitAdminIntegrationTestWithJdbcDatabase
89
extends ConsensusCommitAdminIntegrationTestBase {
10+
private RdbEngineStrategy rdbEngine;
911

1012
@Override
1113
protected Properties getProps(String testName) {
12-
return JdbcEnv.getProperties(testName);
14+
Properties properties = JdbcEnv.getProperties(testName);
15+
rdbEngine = RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties)));
16+
return properties;
1317
}
1418

1519
@Override
1620
protected AdminTestUtils getAdminTestUtils(String testName) {
1721
return new JdbcAdminTestUtils(getProperties(testName));
1822
}
23+
24+
@Override
25+
protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
26+
// "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
27+
// indefinitely) on the Db2 community edition version which we use for the CI.
28+
// However, the index creation is successful on Db2 hosted on IBM Cloud.
29+
// So we disable these tests until the issue with the Db2 community edition is resolved.
30+
return !JdbcTestUtils.isDb2(rdbEngine);
31+
}
1932
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java‎

Lines changed: 115 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public class JdbcAdminImportTestUtils {
114114
"xml",
115115
"geometry",
116116
"geography");
117+
static final List<String> UNSUPPORTED_DATA_TYPES_DB2 =
118+
Arrays.asList("DECIMAL", "DECFLOAT", "XML");
117119

118120
private final RdbEngineStrategy rdbEngine;
119121
private final int majorVersion;
@@ -137,8 +139,10 @@ public List<TestData> createExistingDatabaseWithAllDataTypes(String namespace)
137139
return createExistingOracleDatabaseWithAllDataTypes(namespace);
138140
} else if (JdbcTestUtils.isSqlServer(rdbEngine)) {
139141
return createExistingSqlServerDatabaseWithAllDataTypes(namespace);
142+
} else if (JdbcTestUtils.isDb2(rdbEngine)) {
143+
return createExistingDb2DatabaseWithAllDataTypes(namespace);
140144
} else {
141-
throw new RuntimeException();
145+
throw new AssertionError("Unsupported database engine: " + rdbEngine);
142146
}
143147
}
144148

@@ -455,6 +459,89 @@ private Map<String, Column<?>> prepareInsertColumnsForSqlServer(TableMetadata me
455459
return prepareInsertColumnsWithGenericAndCustomValues(metadata, customColumns);
456460
}
457461

462+
private LinkedHashMap<String, String> prepareColumnsForDb2() {
463+
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
464+
columns.put("pk1", "INT NOT NULL");
465+
columns.put("pk2", "INT NOT NULL");
466+
columns.put("col01", "SMALLINT");
467+
columns.put("col02", "INT");
468+
columns.put("col03", "BIGINT");
469+
columns.put("col04", "REAL");
470+
columns.put("col05", "FLOAT(24)"); // Maps to REAL if precision <=24
471+
columns.put("col06", "DOUBLE");
472+
columns.put("col07", "FLOAT");
473+
columns.put("col08", "FLOAT(25)"); // Maps to DOUBLE if precision => 25
474+
columns.put("col09", "CHAR(3)");
475+
columns.put("col10", "VARCHAR(512)");
476+
columns.put("col11", "CLOB");
477+
columns.put("col12", "GRAPHIC(3)");
478+
columns.put("col13", "VARGRAPHIC(512)");
479+
columns.put("col14", "DBCLOB(5)");
480+
columns.put("col15", "NCHAR(3)");
481+
columns.put("col16", "NVARCHAR(512)");
482+
columns.put("col17", "NCLOB(512)");
483+
columns.put("col18", "BINARY(5)");
484+
columns.put("col19", "VARBINARY(512)");
485+
columns.put("col20", "BLOB(1024)");
486+
columns.put("col21", "CHAR(5) FOR BIT DATA");
487+
columns.put("col22", "VARCHAR(512) FOR BIT DATA");
488+
columns.put("col23", "DATE");
489+
columns.put("col24", "TIME");
490+
columns.put("col25", "TIMESTAMP(6)"); // override to TIME
491+
columns.put("col26", "TIMESTAMP(3)");
492+
columns.put("col27", "TIMESTAMP(3)"); // override to TIMESTAMPTZ
493+
columns.put("col28", "BOOLEAN");
494+
495+
return columns;
496+
}
497+
498+
private Map<String, DataType> prepareOverrideColumnsTypeForDb2() {
499+
return ImmutableMap.of("col25", DataType.TIME, "col27", DataType.TIMESTAMPTZ);
500+
}
501+
502+
private TableMetadata prepareTableMetadataForDb2() {
503+
return TableMetadata.newBuilder()
504+
.addColumn("pk1", DataType.INT)
505+
.addColumn("pk2", DataType.INT)
506+
.addColumn("col01", DataType.INT)
507+
.addColumn("col02", DataType.INT)
508+
.addColumn("col03", DataType.BIGINT)
509+
.addColumn("col04", DataType.FLOAT)
510+
.addColumn("col05", DataType.FLOAT)
511+
.addColumn("col06", DataType.DOUBLE)
512+
.addColumn("col07", DataType.DOUBLE)
513+
.addColumn("col08", DataType.DOUBLE)
514+
.addColumn("col09", DataType.TEXT)
515+
.addColumn("col10", DataType.TEXT)
516+
.addColumn("col11", DataType.TEXT)
517+
.addColumn("col12", DataType.TEXT)
518+
.addColumn("col13", DataType.TEXT)
519+
.addColumn("col14", DataType.TEXT)
520+
.addColumn("col15", DataType.TEXT)
521+
.addColumn("col16", DataType.TEXT)
522+
.addColumn("col17", DataType.TEXT)
523+
.addColumn("col18", DataType.BLOB)
524+
.addColumn("col19", DataType.BLOB)
525+
.addColumn("col20", DataType.BLOB)
526+
.addColumn("col21", DataType.BLOB)
527+
.addColumn("col22", DataType.BLOB)
528+
.addColumn("col23", DataType.DATE)
529+
.addColumn("col24", DataType.TIME)
530+
.addColumn("col25", DataType.TIME)
531+
.addColumn("col26", DataType.TIMESTAMP)
532+
.addColumn("col27", DataType.TIMESTAMPTZ)
533+
.addColumn("col28", DataType.BOOLEAN)
534+
.addPartitionKey("pk1")
535+
.addPartitionKey("pk2")
536+
.build();
537+
}
538+
539+
private Map<String, Column<?>> prepareInsertColumnsForDb2(TableMetadata metadata) {
540+
List<Column<?>> customColumns =
541+
ImmutableList.of(TimeColumn.of("col24", LocalTime.of(11, 8, 35)));
542+
return prepareInsertColumnsWithGenericAndCustomValues(metadata, customColumns);
543+
}
544+
458545
private List<JdbcTestData> prepareCreateNonImportableTableSql(
459546
String namespace, List<String> types) {
460547
ImmutableList.Builder<JdbcTestData> data = new ImmutableList.Builder<>();
@@ -469,7 +556,7 @@ private List<JdbcTestData> prepareCreateNonImportableTableSql(
469556

470557
private String prepareCreateNonImportableTableSql(String namespace, String table, String type) {
471558
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
472-
columns.put("pk", "CHAR(8)");
559+
columns.put("pk", "CHAR(8) NOT NULL");
473560
columns.put("col", type);
474561
return prepareCreateTableSql(
475562
namespace, table, columns, new LinkedHashSet<>(Collections.singletonList("pk")));
@@ -636,6 +723,32 @@ private List<TestData> createExistingSqlServerDatabaseWithAllDataTypes(String na
636723
return ImmutableList.copyOf(data);
637724
}
638725

726+
private List<TestData> createExistingDb2DatabaseWithAllDataTypes(String namespace)
727+
throws SQLException {
728+
List<JdbcTestData> data = new ArrayList<>();
729+
730+
TableMetadata tableMetadata = prepareTableMetadataForDb2();
731+
String sql =
732+
prepareCreateTableSql(
733+
namespace,
734+
SUPPORTED_TABLE_NAME,
735+
prepareColumnsForDb2(),
736+
tableMetadata.getPartitionKeyNames());
737+
data.add(
738+
JdbcTestData.createImportableTable(
739+
SUPPORTED_TABLE_NAME,
740+
sql,
741+
tableMetadata,
742+
prepareOverrideColumnsTypeForDb2(),
743+
prepareInsertColumnsForDb2(tableMetadata)));
744+
745+
data.addAll(prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_DB2));
746+
747+
executeCreateTableSql(data);
748+
749+
return ImmutableList.copyOf(data);
750+
}
751+
639752
private void executeCreateTableSql(List<JdbcTestData> data) throws SQLException {
640753
String[] sqls = data.stream().map(JdbcTestData::getCreateTableSql).toArray(String[]::new);
641754
execute(sqls);

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,30 @@
11
package com.scalar.db.storage.jdbc;
22

33
import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
4+
import com.scalar.db.config.DatabaseConfig;
45
import com.scalar.db.util.AdminTestUtils;
56
import java.util.Properties;
67

78
public class JdbcAdminIntegrationTest extends DistributedStorageAdminIntegrationTestBase {
9+
private RdbEngineStrategy rdbEngine;
810

911
@Override
1012
protected Properties getProperties(String testName) {
11-
return JdbcEnv.getProperties(testName);
13+
Properties properties = JdbcEnv.getProperties(testName);
14+
rdbEngine = RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties)));
15+
return properties;
1216
}
1317

1418
@Override
1519
protected AdminTestUtils getAdminTestUtils(String testName) {
1620
return new JdbcAdminTestUtils(getProperties(testName));
1721
}
22+
23+
@Override
24+
protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
25+
// "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
26+
// indefinitely) on Db2 community edition version but works on Db2 hosted on IBM Cloud.
27+
// So we disable these tests until the issue is resolved.
28+
return !JdbcTestUtils.isDb2(rdbEngine);
29+
}
1830
}

0 commit comments

Comments
 (0)