feat(java/driver/jdbc): add hooks for JDBC type system mapping (#722)

- Adds a custom option to the ADBC JDBC adapter to provide a custom type
mapper from JDBC column info to ArrowType
- Adds a customized JdbcFieldInfo that exposes more fields that are
necessary (e.g. PostgreSQL's JDBC driver exposes `TIMESTAMP WITHOUT TIME
ZONE` and `TIMESTAMP WITH TIME ZONE` as `Types.TIMESTAMP`, so you have
to look at the type name instead)
- Future work: #728
- Future work: apache/arrow#35916
- Future work: #727
- Add (failing) tests demonstrating that the values read are
inconsistent with the assumed types

Fixes #720.

java/driver/jdbc-validation-mssqlserver/pom.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>arrow-adbc-java-root</artifactId>
+ <groupId>org.apache.arrow.adbc</groupId>
+ <version>0.5.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>adbc-driver-jdbc-validation-mssqlserver</artifactId>
+ <packaging>jar</packaging>
+ <name>Arrow ADBC Driver JDBC Validation with Microsoft SQL Server</name>
+ <description>Tests validating the JDBC driver against Microsoft SQL Server.</description>
+
+ <properties>
+ <maven.deploy.skip>true</maven.deploy.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.arrow.adbc</groupId>
+ <artifactId>adbc-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow.adbc</groupId>
+ <artifactId>adbc-driver-jdbc</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ <version>12.2.0.jre11</version>
+ </dependency>
+
+ <!-- Testing -->
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow.adbc</groupId>
+ <artifactId>adbc-driver-validation</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>

java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adbc.driver.jdbc.mssqlserver;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.arrow.adbc.core.AdbcDatabase;
+import org.apache.arrow.adbc.core.AdbcDriver;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.driver.jdbc.JdbcDriver;
+import org.apache.arrow.adbc.driver.jdbc.adapter.JdbcToArrowTypeConverters;
+import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.junit.jupiter.api.Assumptions;
+
+public class MsSqlServerQuirks extends SqlValidationQuirks {
+ static final String URL_ENV_VAR = "ADBC_JDBC_MSSQL_URL";
+ static final String USER_ENV_VAR = "ADBC_JDBC_MSSQL_USER";
+ static final String PASSWORD_ENV_VAR = "ADBC_JDBC_MSSQL_PASSWORD";
+
+ static String makeJdbcUrl() {
+ final String url = System.getenv(URL_ENV_VAR);
+ final String user = System.getenv(USER_ENV_VAR);
+ final String password = System.getenv(PASSWORD_ENV_VAR);
+ Assumptions.assumeFalse(url == null, "Microsoft SQL Server not found, set " + URL_ENV_VAR);
+ Assumptions.assumeFalse(url.isEmpty(), "Microsoft SQL Server not found, set " + URL_ENV_VAR);
+ return String.format(
+ "jdbc:sqlserver://%s;user=%s;password=%s;trustServerCertificate=true", url, user, password);
+ }
+
+ @Override
+ public AdbcDatabase initDatabase(BufferAllocator allocator) throws AdbcException {
+ String url = makeJdbcUrl();
+
+ final Map<String, Object> parameters = new HashMap<>();
+ parameters.put(AdbcDriver.PARAM_URL, url);
+ parameters.put(
+ JdbcDriver.PARAM_JDBC_TO_ARROW_TYPE, JdbcToArrowTypeConverters.MICROSOFT_SQL_SERVER);
+ return new JdbcDriver(allocator).open(parameters);
+ }
+
+ @Override
+ public void cleanupTable(String name) throws Exception {
+ try (final Connection connection1 = DriverManager.getConnection(makeJdbcUrl())) {
+ try (Statement statement = connection1.createStatement()) {
+ statement.execute("DROP TABLE " + name);
+ } catch (SQLException ignored) {
+ }
+ }
+ }
+
+ @Override
+ public String defaultCatalog() {
+ // XXX: this should really come from configuration
+ return "msdb";
+ }
+
+ @Override
+ public String caseFoldTableName(String name) {
+ return name.toLowerCase();
+ }
+
+ @Override
+ public String caseFoldColumnName(String name) {
+ return name.toLowerCase();
+ }
+
+ @Override
+ public TimeUnit defaultTimestampUnit() {
+ return TimeUnit.NANOSECOND;
+ }
+}

java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerSqlTypeTest.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adbc.driver.jdbc.mssqlserver;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.apache.arrow.adbc.driver.testsuite.AbstractSqlTypeTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class MsSqlServerSqlTypeTest extends AbstractSqlTypeTest {
+ @BeforeAll
+ public static void beforeAll() {
+ quirks = new MsSqlServerQuirks();
+ }
+
+ @Test
+ @Override
+ protected void timestampWithoutTimeZoneValue() {
+ // TODO(https://github.com/apache/arrow/issues/35916): needs upstream fix
+ // XXX: Java 8 compiler complains without lambda https://stackoverflow.com/questions/33621060
+ //noinspection Convert2MethodRef
+ assertThrows(RuntimeException.class, () -> super.timestampWithTimeZoneValue());
+ }
+
+ @Test
+ @Override
+ protected void timestampWithTimeZoneValue() {
+ // TODO(https://github.com/apache/arrow/issues/35916): needs upstream fix
+ // XXX: Java 8 compiler complains without lambda https://stackoverflow.com/questions/33621060
+ //noinspection Convert2MethodRef
+ assertThrows(RuntimeException.class, () -> super.timestampWithTimeZoneValue());
+ }
+}

java/driver/jdbc-validation-mssqlserver/src/test/resources/MsSqlServerSqlTypeTest.sql

@@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS adbc_alltypes;
+
+CREATE TABLE adbc_alltypes (
+ bigint_t BIGINT,
+ date_t DATE,
+ int_t INT,
+ text_t TEXT,
+ time_without_time_zone_t TIME,
+ timestamp_without_time_zone_t DATETIME2,
+ timestamp_with_time_zone_t DATETIMEOFFSET
+);
+
+INSERT INTO adbc_alltypes VALUES (
+ 42,
+ '2000-01-01',
+ 42,
+ 'foo',
+ '04:05:06.789012',
+ '2000-01-02T03:04:05.123456',
+ '2000-01-02T03:04:05.123456+06:00'
+), (
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL
+);

java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgreSqlTypeTest.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adbc.driver.jdbc.postgresql;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.apache.arrow.adbc.driver.testsuite.AbstractSqlTypeTest;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class PostgreSqlTypeTest extends AbstractSqlTypeTest {
+ @BeforeAll
+ public static void beforeAll() {
+ quirks = new PostgresqlQuirks();
+ }
+
+ @Test
+ @Override
+ protected void timestampWithoutTimeZoneValue() {
+ // TODO(https://github.com/apache/arrow/issues/35916): needs upstream fix
+ // XXX: Java 8 compiler complains without lambda https://stackoverflow.com/questions/33621060
+ //noinspection Convert2MethodRef
+ assertThrows(AssertionError.class, () -> super.timestampWithTimeZoneValue());
+ }
+
+ @Test
+ @Override
+ protected void timestampWithTimeZoneValue() {
+ // TODO(https://github.com/apache/arrow/issues/35916): needs upstream fix
+ // XXX: Java 8 compiler complains without lambda https://stackoverflow.com/questions/33621060
+ //noinspection Convert2MethodRef
+ assertThrows(AssertionError.class, () -> super.timestampWithTimeZoneValue());
+ }
+
+ @Test
+ protected void timestamp6WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p6_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null));
+ }
+
+ @Test
+ protected void timestamp5WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p5_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null));
+ }
+
+ @Test
+ protected void timestamp4WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p4_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null));
+ }
+
+ @Test
+ protected void timestamp3WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p3_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null));
+ }
+
+ @Test
+ protected void timestamp2WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p2_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null));
+ }
+
+ @Test
+ protected void timestamp1WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p1_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null));
+ }
+
+ @Test
+ protected void timestamp0WithoutTimeZoneType() throws Exception {
+ final Schema schema = connection.getTableSchema(null, null, "adbc_alltypes");
+ assertThat(schema.findField("timestamp_without_time_zone_p0_t").getType())
+ .isEqualTo(new ArrowType.Timestamp(TimeUnit.SECOND, null));
+ }
+}

java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java

@@ -27,9 +27,11 @@
 import org.apache.arrow.adbc.core.AdbcDriver;
 import org.apache.arrow.adbc.core.AdbcException;
 import org.apache.arrow.adbc.driver.jdbc.JdbcDriver;
+import org.apache.arrow.adbc.driver.jdbc.adapter.JdbcToArrowTypeConverters;
 import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks;
 import org.apache.arrow.adbc.sql.SqlQuirks;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.junit.jupiter.api.Assumptions;
 
@@ -66,6 +68,7 @@ public AdbcDatabase initDatabase(BufferAllocator allocator) throws AdbcException
  return SqlQuirks.DEFAULT_ARROW_TYPE_TO_SQL_TYPE_NAME_MAPPING.apply(arrowType);
  }))
  .build());
+ parameters.put(JdbcDriver.PARAM_JDBC_TO_ARROW_TYPE, JdbcToArrowTypeConverters.POSTGRESQL);
  return new JdbcDriver(allocator).open(parameters);
  }
 
@@ -94,4 +97,9 @@ public String caseFoldTableName(String name) {
  public String caseFoldColumnName(String name) {
  return name.toLowerCase();
  }
+
+ @Override
+ public TimeUnit defaultTimestampUnit() {
+ return TimeUnit.MICROSECOND;
+ }
 }