Skip to content

Commit 1ea90ab

Browse files
feeblefakieTorch3333
feeblefakie
and
Torch3333
authored
Backport to branch(3) : Support IBM Db2 (#2636)
Co-authored-by: Vincent Guilpain <vincent.guilpain@scalar-labs.com>
1 parent f21eb24 commit 1ea90ab

File tree

39 files changed

+1739
-119
lines changed

39 files changed

+1739
-119
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1578,6 +1578,162 @@ jobs:
15781578
name: yugabytedb_2_integration_test_reports_${{ matrix.mode.label }}
15791579
path: core/build/reports/tests/integrationTestJdbc
15801580

1581+
integration-test-for-jdbc-db2-11-5:
1582+
name: Db2 11.5 integration test (${{ matrix.mode.label }})
1583+
runs-on: ubuntu-latest
1584+
1585+
services:
1586+
db2:
1587+
image: icr.io/db2_community/db2:11.5.9.0
1588+
env:
1589+
DB2INSTANCE: db2inst1
1590+
DB2INST1_PASSWORD: db2inst1
1591+
DBNAME: test_db
1592+
LICENSE: accept
1593+
ports:
1594+
- 50000:50000
1595+
options: --privileged --name db2
1596+
1597+
strategy:
1598+
matrix:
1599+
mode:
1600+
- label: default
1601+
group_commit_enabled: false
1602+
- label: with_group_commit
1603+
group_commit_enabled: true
1604+
1605+
steps:
1606+
- uses: actions/checkout@v4
1607+
1608+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
1609+
uses: actions/setup-java@v4
1610+
with:
1611+
java-version: ${{ env.JAVA_VERSION }}
1612+
distribution: ${{ env.JAVA_VENDOR }}
1613+
1614+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
1615+
uses: actions/setup-java@v4
1616+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
1617+
with:
1618+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
1619+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
1620+
1621+
- name: Login to Oracle container registry
1622+
uses: docker/login-action@v3
1623+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1624+
with:
1625+
registry: container-registry.oracle.com
1626+
username: ${{ secrets.OCR_USERNAME }}
1627+
password: ${{ secrets.OCR_TOKEN }}
1628+
1629+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
1630+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1631+
run: |
1632+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
1633+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
1634+
1635+
- name: Setup Gradle
1636+
uses: gradle/actions/setup-gradle@v4
1637+
1638+
- name: Wait for the container to be ready
1639+
timeout-minutes: 10
1640+
run: |
1641+
while ! docker logs db2 2>&1 | grep -q "Setup has completed"
1642+
do
1643+
echo "Container is not yet ready"
1644+
sleep 5s
1645+
done
1646+
echo "Container is ready"
1647+
1648+
- name: Execute Gradle 'integrationTestJdbc' task
1649+
run: ./gradlew integrationTestJdbc -Dscalardb.jdbc.url="jdbc:db2://localhost:50000/test_db" -Dscalardb.jdbc.username=db2inst1 -Dscalardb.jdbc.password=db2inst1 ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
1650+
1651+
- name: Upload Gradle test reports
1652+
if: always()
1653+
uses: actions/upload-artifact@v4
1654+
with:
1655+
name: db2_11.5_integration_test_reports_${{ matrix.mode.label }}
1656+
path: core/build/reports/tests/integrationTestJdbc
1657+
1658+
integration-test-for-jdbc-db2-12-1:
1659+
name: Db2 12.1 integration test (${{ matrix.mode.label }})
1660+
runs-on: ubuntu-latest
1661+
1662+
services:
1663+
db2:
1664+
image: icr.io/db2_community/db2:12.1.1.0
1665+
env:
1666+
DB2INSTANCE: db2inst1
1667+
DB2INST1_PASSWORD: db2inst1
1668+
DBNAME: test_db
1669+
LICENSE: accept
1670+
ports:
1671+
- 50000:50000
1672+
options: --privileged --name db2
1673+
1674+
strategy:
1675+
matrix:
1676+
mode:
1677+
- label: default
1678+
group_commit_enabled: false
1679+
- label: with_group_commit
1680+
group_commit_enabled: true
1681+
1682+
steps:
1683+
- uses: actions/checkout@v4
1684+
1685+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
1686+
uses: actions/setup-java@v4
1687+
with:
1688+
java-version: ${{ env.JAVA_VERSION }}
1689+
distribution: ${{ env.JAVA_VENDOR }}
1690+
1691+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
1692+
uses: actions/setup-java@v4
1693+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
1694+
with:
1695+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
1696+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
1697+
1698+
- name: Login to Oracle container registry
1699+
uses: docker/login-action@v3
1700+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1701+
with:
1702+
registry: container-registry.oracle.com
1703+
username: ${{ secrets.OCR_USERNAME }}
1704+
password: ${{ secrets.OCR_TOKEN }}
1705+
1706+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
1707+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
1708+
run: |
1709+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
1710+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
1711+
1712+
- name: Setup Gradle
1713+
uses: gradle/actions/setup-gradle@v4
1714+
1715+
- name: Wait for the container to be ready
1716+
timeout-minutes: 10
1717+
run: |
1718+
while ! docker logs db2 2>&1 | grep -q "Setup has completed"
1719+
do
1720+
echo "Container is not yet ready"
1721+
sleep 5s
1722+
done
1723+
echo "Container is ready"
1724+
1725+
- name: Execute Gradle 'integrationTestJdbc' task
1726+
run: ./gradlew integrationTestJdbc -Dscalardb.jdbc.url="jdbc:db2://localhost:50000/test_db" -Dscalardb.jdbc.username=db2inst1 -Dscalardb.jdbc.password=db2inst1 ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
1727+
1728+
- name: Upload Gradle test reports
1729+
if: always()
1730+
uses: actions/upload-artifact@v4
1731+
with:
1732+
name: db2_12.1_integration_test_reports_${{ matrix.mode.label }}
1733+
path: core/build/reports/tests/integrationTestJdbc
1734+
1735+
1736+
15811737
integration-test-for-multi-storage:
15821738
name: Multi-storage integration test (${{ matrix.mode.label }})
15831739
runs-on: ubuntu-latest

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ subprojects {
3535
sqlserverDriverVersion = '12.8.1.jre8'
3636
sqliteDriverVersion = '3.49.1.0'
3737
yugabyteDriverVersion = '42.7.3-yb-4'
38+
db2DriverVersion= '12.1.0.0'
3839
mariadDbDriverVersion = '3.5.3'
3940
picocliVersion = '4.7.7'
4041
commonsTextVersion = '1.13.1'

‎core/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ dependencies {
107107
implementation "com.microsoft.sqlserver:mssql-jdbc:${sqlserverDriverVersion}"
108108
implementation "org.xerial:sqlite-jdbc:${sqliteDriverVersion}"
109109
implementation "com.yugabyte:jdbc-yugabytedb:${yugabyteDriverVersion}"
110+
implementation "com.ibm.db2:jcc:${db2DriverVersion}"
110111
implementation ("org.mariadb.jdbc:mariadb-java-client:${mariadDbDriverVersion}") {
111112
exclude group: 'org.slf4j', module: 'slf4j-api'
112113
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,13 @@
1212
public class ConsensusCommitAdminIntegrationTestWithJdbcDatabase
1313
extends ConsensusCommitAdminIntegrationTestBase {
1414

15+
private RdbEngineStrategy rdbEngine;
16+
1517
@Override
1618
protected Properties getProps(String testName) {
17-
return ConsensusCommitJdbcEnv.getProperties(testName);
19+
Properties properties = JdbcEnv.getProperties(testName);
20+
rdbEngine = RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties)));
21+
return properties;
1822
}
1923

2024
@Override
@@ -110,4 +114,13 @@ public void namespaceExists_ShouldReturnCorrectResults() throws ExecutionExcepti
110114
public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentException() {
111115
super.createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentException();
112116
}
117+
118+
@Override
119+
protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
120+
// "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
121+
// indefinitely) on the Db2 community edition docker version which we use for the CI.
122+
// However, the index creation is successful on Db2 hosted on IBM Cloud.
123+
// So we disable these tests until the issue with the Db2 community edition is resolved.
124+
return !JdbcTestUtils.isDb2(rdbEngine);
125+
}
113126
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java‎

Lines changed: 115 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public class JdbcAdminImportTestUtils {
114114
"xml",
115115
"geometry",
116116
"geography");
117+
static final List<String> UNSUPPORTED_DATA_TYPES_DB2 =
118+
Arrays.asList("DECIMAL", "DECFLOAT", "XML");
117119

118120
private final JdbcConfig config;
119121
private final RdbEngineStrategy rdbEngine;
@@ -135,8 +137,10 @@ public List<TestData> createExistingDatabaseWithAllDataTypes(String namespace)
135137
return createExistingOracleDatabaseWithAllDataTypes(namespace);
136138
} else if (JdbcTestUtils.isSqlServer(rdbEngine)) {
137139
return createExistingSqlServerDatabaseWithAllDataTypes(namespace);
140+
} else if (JdbcTestUtils.isDb2(rdbEngine)) {
141+
return createExistingDb2DatabaseWithAllDataTypes(namespace);
138142
} else {
139-
throw new RuntimeException();
143+
throw new AssertionError("Unsupported database engine: " + rdbEngine);
140144
}
141145
}
142146

@@ -459,6 +463,89 @@ private Map<String, Column<?>> prepareInsertColumnsForSqlServer(TableMetadata me
459463
return prepareInsertColumnsWithGenericAndCustomValues(metadata, customColumns);
460464
}
461465

466+
private LinkedHashMap<String, String> prepareColumnsForDb2() {
467+
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
468+
columns.put("pk1", "INT NOT NULL");
469+
columns.put("pk2", "INT NOT NULL");
470+
columns.put("col01", "SMALLINT");
471+
columns.put("col02", "INT");
472+
columns.put("col03", "BIGINT");
473+
columns.put("col04", "REAL");
474+
columns.put("col05", "FLOAT(24)"); // Maps to REAL if precision <=24
475+
columns.put("col06", "DOUBLE");
476+
columns.put("col07", "FLOAT");
477+
columns.put("col08", "FLOAT(25)"); // Maps to DOUBLE if precision => 25
478+
columns.put("col09", "CHAR(3)");
479+
columns.put("col10", "VARCHAR(512)");
480+
columns.put("col11", "CLOB");
481+
columns.put("col12", "GRAPHIC(3)");
482+
columns.put("col13", "VARGRAPHIC(512)");
483+
columns.put("col14", "DBCLOB(5)");
484+
columns.put("col15", "NCHAR(3)");
485+
columns.put("col16", "NVARCHAR(512)");
486+
columns.put("col17", "NCLOB(512)");
487+
columns.put("col18", "BINARY(5)");
488+
columns.put("col19", "VARBINARY(512)");
489+
columns.put("col20", "BLOB(1024)");
490+
columns.put("col21", "CHAR(5) FOR BIT DATA");
491+
columns.put("col22", "VARCHAR(512) FOR BIT DATA");
492+
columns.put("col23", "DATE");
493+
columns.put("col24", "TIME");
494+
columns.put("col25", "TIMESTAMP(6)"); // override to TIME
495+
columns.put("col26", "TIMESTAMP(3)");
496+
columns.put("col27", "TIMESTAMP(3)"); // override to TIMESTAMPTZ
497+
columns.put("col28", "BOOLEAN");
498+
499+
return columns;
500+
}
501+
502+
private Map<String, DataType> prepareOverrideColumnsTypeForDb2() {
503+
return ImmutableMap.of("col25", DataType.TIME, "col27", DataType.TIMESTAMPTZ);
504+
}
505+
506+
private TableMetadata prepareTableMetadataForDb2() {
507+
return TableMetadata.newBuilder()
508+
.addColumn("pk1", DataType.INT)
509+
.addColumn("pk2", DataType.INT)
510+
.addColumn("col01", DataType.INT)
511+
.addColumn("col02", DataType.INT)
512+
.addColumn("col03", DataType.BIGINT)
513+
.addColumn("col04", DataType.FLOAT)
514+
.addColumn("col05", DataType.FLOAT)
515+
.addColumn("col06", DataType.DOUBLE)
516+
.addColumn("col07", DataType.DOUBLE)
517+
.addColumn("col08", DataType.DOUBLE)
518+
.addColumn("col09", DataType.TEXT)
519+
.addColumn("col10", DataType.TEXT)
520+
.addColumn("col11", DataType.TEXT)
521+
.addColumn("col12", DataType.TEXT)
522+
.addColumn("col13", DataType.TEXT)
523+
.addColumn("col14", DataType.TEXT)
524+
.addColumn("col15", DataType.TEXT)
525+
.addColumn("col16", DataType.TEXT)
526+
.addColumn("col17", DataType.TEXT)
527+
.addColumn("col18", DataType.BLOB)
528+
.addColumn("col19", DataType.BLOB)
529+
.addColumn("col20", DataType.BLOB)
530+
.addColumn("col21", DataType.BLOB)
531+
.addColumn("col22", DataType.BLOB)
532+
.addColumn("col23", DataType.DATE)
533+
.addColumn("col24", DataType.TIME)
534+
.addColumn("col25", DataType.TIME)
535+
.addColumn("col26", DataType.TIMESTAMP)
536+
.addColumn("col27", DataType.TIMESTAMPTZ)
537+
.addColumn("col28", DataType.BOOLEAN)
538+
.addPartitionKey("pk1")
539+
.addPartitionKey("pk2")
540+
.build();
541+
}
542+
543+
private Map<String, Column<?>> prepareInsertColumnsForDb2(TableMetadata metadata) {
544+
List<Column<?>> customColumns =
545+
ImmutableList.of(TimeColumn.of("col24", LocalTime.of(11, 8, 35)));
546+
return prepareInsertColumnsWithGenericAndCustomValues(metadata, customColumns);
547+
}
548+
462549
private List<JdbcTestData> prepareCreateNonImportableTableSql(
463550
String namespace, List<String> types) {
464551
ImmutableList.Builder<JdbcTestData> data = new ImmutableList.Builder<>();
@@ -473,7 +560,7 @@ private List<JdbcTestData> prepareCreateNonImportableTableSql(
473560

474561
private String prepareCreateNonImportableTableSql(String namespace, String table, String type) {
475562
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
476-
columns.put("pk", "CHAR(8)");
563+
columns.put("pk", "CHAR(8) NOT NULL");
477564
columns.put("col", type);
478565
return prepareCreateTableSql(
479566
namespace, table, columns, new LinkedHashSet<>(Collections.singletonList("pk")));
@@ -640,6 +727,32 @@ private List<TestData> createExistingSqlServerDatabaseWithAllDataTypes(String na
640727
return ImmutableList.copyOf(data);
641728
}
642729

730+
private List<TestData> createExistingDb2DatabaseWithAllDataTypes(String namespace)
731+
throws SQLException {
732+
List<JdbcTestData> data = new ArrayList<>();
733+
734+
TableMetadata tableMetadata = prepareTableMetadataForDb2();
735+
String sql =
736+
prepareCreateTableSql(
737+
namespace,
738+
SUPPORTED_TABLE_NAME,
739+
prepareColumnsForDb2(),
740+
tableMetadata.getPartitionKeyNames());
741+
data.add(
742+
JdbcTestData.createImportableTable(
743+
SUPPORTED_TABLE_NAME,
744+
sql,
745+
tableMetadata,
746+
prepareOverrideColumnsTypeForDb2(),
747+
prepareInsertColumnsForDb2(tableMetadata)));
748+
749+
data.addAll(prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_DB2));
750+
751+
executeCreateTableSql(data);
752+
753+
return ImmutableList.copyOf(data);
754+
}
755+
643756
private void executeCreateTableSql(List<JdbcTestData> data) throws SQLException {
644757
String[] sqls = data.stream().map(JdbcTestData::getCreateTableSql).toArray(String[]::new);
645758
execute(sqls);

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@
88
import org.junit.jupiter.api.condition.DisabledIf;
99

1010
public class JdbcAdminIntegrationTest extends DistributedStorageAdminIntegrationTestBase {
11+
private RdbEngineStrategy rdbEngine;
1112

1213
@Override
1314
protected Properties getProperties(String testName) {
14-
return JdbcEnv.getProperties(testName);
15+
Properties properties = JdbcEnv.getProperties(testName);
16+
rdbEngine = RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties)));
17+
return properties;
1518
}
1619

1720
@Override
@@ -94,4 +97,12 @@ public void namespaceExists_ShouldReturnCorrectResults() throws ExecutionExcepti
9497
public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentException() {
9598
super.createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentException();
9699
}
100+
101+
@Override
102+
protected boolean isCreateIndexOnTextAndBlobColumnsEnabled() {
103+
// "admin.createIndex()" for TEXT and BLOB columns fails (the "create index" query runs
104+
// indefinitely) on Db2 community edition version but works on Db2 hosted on IBM Cloud.
105+
// So we disable these tests until the issue is resolved.
106+
return !JdbcTestUtils.isDb2(rdbEngine);
107+
}
97108
}

0 commit comments

Comments
 (0)