From 0842464f65b021c2b87a750f702d0402eaf5b0a1 Mon Sep 17 00:00:00 2001 From: linghengqian Date: Sun, 24 Nov 2024 18:01:20 +0800 Subject: [PATCH] Adds documentation for ClickHouse support --- RELEASE-NOTES.md | 1 + .../graalvm-native-image/_index.cn.md | 13 +- .../graalvm-native-image/_index.en.md | 13 +- .../optional-plugins/clickhouse/_index.cn.md | 217 +++++++++++++++++ .../optional-plugins/clickhouse/_index.en.md | 222 ++++++++++++++++++ .../optional-plugins/hiveserver2/_index.cn.md | 4 + .../optional-plugins/hiveserver2/_index.en.md | 4 + .../testcontainers/_index.cn.md | 4 + .../testcontainers/_index.en.md | 4 + .../reflect-config.json | 18 +- .../resource-config.json | 2 +- test/native/pom.xml | 4 + .../natived/commons/TestShardingService.java | 57 ++--- .../jdbc/databases/ClickHouseTest.java | 3 +- 14 files changed, 506 insertions(+), 60 deletions(-) create mode 100644 docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md create mode 100644 docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 911bcde21bcf6..dc5ba998046d9 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -26,6 +26,7 @@ 1. Infra: Support setting `hive_conf_list`, `hive_var_list` and `sess_var_list` for jdbcURL when connecting to HiveServer2 - [#33749](https://github.com/apache/shardingsphere/pull/33749) 1. Infra: Support connecting to HiveServer2 through database connection pools other than HikariCP - [#33762](https://github.com/apache/shardingsphere/pull/33762) 1. Proxy Native: Support connecting to HiveServer2 with ZooKeeper Service Discovery enabled in GraalVM Native Image - [#33768](https://github.com/apache/shardingsphere/pull/33768) +1. Doc: Adds documentation for ClickHouse support - [#33779](https://github.com/apache/shardingsphere/pull/33779) ### Bug Fixes diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md index d4f2a24d928af..ac678804be491 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.cn.md @@ -264,6 +264,7 @@ Caused by: java.io.UnsupportedEncodingException: Codepage Cp1252 is not supporte 原则上，ShardingSphere 的 GraalVM Native Image 集成不希望使用 classifier 为 `all` 的 `com.clickhouse:clickhouse-jdbc`， 因为 Uber Jar 会导致采集重复的 GraalVM Reachability Metadata。 可能的配置例子如下， + ```xml @@ -287,8 +288,6 @@ Caused by: java.io.UnsupportedEncodingException: Codepage Cp1252 is not supporte ``` -ClickHouse 不支持 ShardingSphere 集成级别的本地事务，XA 事务和 Seata AT 模式事务，更多讨论位于 https://github.com/ClickHouse/clickhouse-docs/issues/2300 。 - 7. 受 https://github.com/grpc/grpc-java/issues/10601 影响，用户如果在项目中引入了 `org.apache.hive:hive-jdbc`， 则需要在项目的 classpath 的 `META-INF/native-image/io.grpc/grpc-netty-shaded` 文件夹下创建包含如下内容的文件 `native-image.properties`， @@ -401,10 +400,16 @@ cd ./shardingsphere/ "resources":{ "includes":[{ "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, - "pattern":"\\QcustomPath/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql//global.yaml\\E" + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql//global.yaml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql/\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/features/sharding//global.yaml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, - "pattern":"\\QcustomPath/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql/\\E" + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/features/sharding/\\E" }]}, "bundles":[] } diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md index 38bfea154192d..680f492bd3fa3 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/graalvm-native-image/_index.en.md @@ -277,6 +277,7 @@ users need to manually introduce the relevant optional modules and the ClickHous In principle, ShardingSphere's GraalVM Native Image integration does not want to use `com.clickhouse:clickhouse-jdbc` with classifier `all`, because Uber Jar will cause the collection of duplicate GraalVM Reachability Metadata. Possible configuration examples are as follows, + ```xml @@ -299,8 +300,6 @@ Possible configuration examples are as follows, ``` -ClickHouse does not support local transactions, XA transactions, and Seata AT mode transactions at the ShardingSphere integration level. -More discussion is at https://github.com/ClickHouse/clickhouse-docs/issues/2300 . 7. Affected by https://github.com/grpc/grpc-java/issues/10601 , should users incorporate `org.apache.hive:hive-jdbc` into their project, it is imperative to create a file named `native-image.properties` within the directory `META-INF/native-image/io.grpc/grpc-netty-shaded` of the classpath, @@ -417,10 +416,16 @@ similar to the following. "resources":{ "includes":[{ "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, - "pattern":"\\QcustomPath/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql//global.yaml\\E" + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql//global.yaml\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql/\\E" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/features/sharding//global.yaml\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.config.ProxyConfigurationLoader"}, - "pattern":"\\QcustomPath/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/databases/postgresql/\\E" + "pattern":"\\Qhome/runner/work/shardingsphere/shardingsphere/test/native/src/test/resources/test-native/yaml/proxy/features/sharding/\\E" }]}, "bundles":[] } diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md new file mode 100644 index 0000000000000..cc401c672aecc --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md @@ -0,0 +1,217 @@ ++++ +title = "ClickHouse" +weight = 6 ++++ + +## 背景信息 + +ShardingSphere 默认情况下不提供对 `com.clickhouse.jdbc.ClickHouseDriver` 的 `driverClassName` 的支持。 +ShardingSphere 对 ClickHouse JDBC Driver 的支持位于可选模块中。 + +## 前提条件 + +要在 ShardingSphere 的配置文件为数据节点使用类似 `jdbc:ch://localhost:8123/demo_ds_0` 的 `jdbcUrl`， +可能的 Maven 依赖关系如下， + +```xml + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-clickhouse + ${shardingsphere.version} + + + com.clickhouse + clickhouse-jdbc + http + 0.6.3 + + +``` + +## 配置示例 + +### 启动 ClickHouse + +编写 Docker Compose 文件来启动 ClickHouse。 + +```yaml +services: + clickhouse-server: + image: clickhouse/clickhouse-server:24.10.2.80 + ports: + - "8123:8123" +``` + +### 创建业务表 + +通过第三方工具在 ClickHouse 内创建业务库与业务表。 +以 DBeaver Community 为例，若使用 Ubuntu 22.04.4，可通过 Snapcraft 快速安装， + +```shell +sudo apt update && sudo apt upgrade -y +sudo snap install dbeaver-ce +snap run dbeaver-ce +``` + +在 DBeaver Community 内，使用 `jdbc:ch://localhost:8123/default` 的 `jdbcUrl`，`default` 的`username` 连接至 ClickHouse， +`password` 留空。 +执行如下 SQL， + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +CREATE DATABASE demo_ds_0; +CREATE DATABASE demo_ds_1; +CREATE DATABASE demo_ds_2; +``` + +分别使用 `jdbc:ch://localhost:8123/demo_ds_0` ， +`jdbc:ch://localhost:8123/demo_ds_1` 和 `jdbc:ch://localhost:8123/demo_ds_2` 的 `jdbcUrl` 连接至 ClickHouse 来执行如下 SQL， + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +create table IF NOT EXISTS t_order ( + order_id Int64 NOT NULL DEFAULT rand(), + order_type Int32, + user_id Int32 NOT NULL, + address_id Int64 NOT NULL, + status String +) engine = MergeTree + primary key (order_id) + order by (order_id); + +TRUNCATE TABLE t_order; +``` + +### 在业务项目创建 ShardingSphere 数据源 + +在业务项目引入`前提条件`涉及的依赖后，在业务项目的 classpath 上编写 ShardingSphere 数据源的配置文件`demo.yaml`， + +```yaml +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_0 + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_1 + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_2 +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: INLINE + props: + algorithm-expression: ds_${user_id % 2} + keyGenerators: + snowflake: + type: SNOWFLAKE +``` + +### 享受集成 + +创建 ShardingSphere 的数据源以享受集成， + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +public class ExampleUtils { + void test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')"); + statement.executeQuery("SELECT * FROM t_order"); + statement.execute("alter table t_order delete where order_id=1"); + } + } +} +``` + +## 使用限制 + +### SQL 限制 + +ShardingSphere JDBC DataSource 尚不支持执行 ClickHouse 的 `CREATE TABLE` 语句和 `TRUNCATE TABLE` 语句。 +用户应考虑为 ShardingSphere 提交包含单元测试的 PR。 + +### 分布式序列限制 + +ClickHouse 自身的，对应分布式序列功能的列类型是 `UUID`，`UUID` 在 ClickHouse JDBC Driver 中接收为 `java.util.UUID`， +参考 https://github.com/ClickHouse/ClickHouse/issues/56228 。 +而 ShardingSphere 的 `SNOWFLAKE` 的分布式序列 SPI 实现对应的列类型是 `UInt64`， +在 ShardingSphere JDBC Driver 中接收为 `java.lang.Long`。 + +当为 ShardingSphere 配置连接至 ClickHouse 时， 若同时配置了 ShardingSphere 使用 `SNOWFLAKE` 的分布式序列 SPI 实现， +ShardingSphere 的分布式序列功能使用的 ClickHouse 真实数据库中的列类型不应该被设置为 `UUID`。 + +由于 `com.clickhouse:clickhouse-jdbc:0.6.3:http` Maven 模块的 `com.clickhouse.jdbc.ClickHouseConnection#prepareStatement(String, int)` +故意在 `autoGeneratedKeys` 为 `java.sql.Statement.RETURN_GENERATED_KEYS` 时抛出异常， +以阻止 ShardingSphere 正常代理 `com.clickhouse.jdbc.internal.ClickHouseConnectionImpl`， +因此如果用户需要从 JDBC 业务代码获取 ShardingSphere 生成的分布式序列，需要将 `autoGeneratedKeys` 置为 `java.sql.Statement.NO_GENERATED_KEYS`。 + +一个可能的示例如下， + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.*; +public class ExampleTest { + long test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')", + Statement.NO_GENERATED_KEYS + )) { + preparedStatement.executeUpdate(); + try (ResultSet resultSet = preparedStatement.getGeneratedKeys()) { + if (resultSet.next()) { + return resultSet.getLong(1); + } + throw new RuntimeException(); + } + } + } +} +``` + +### 事务限制 + +ClickHouse 不支持 ShardingSphere 集成级别的本地事务，XA 事务和 Seata AT 模式事务， +更多讨论位于 https://github.com/ClickHouse/clickhouse-docs/issues/2300 。 + +### 嵌入式 ClickHouse 限制 + +嵌入式 ClickHouse `chDB` 尚未发布 Java 客户端， +ShardingSphere 不针对 SNAPSHOT 版本的 https://github.com/chdb-io/chdb-java 做集成测试。 +参考 https://github.com/chdb-io/chdb/issues/243 。 diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md new file mode 100644 index 0000000000000..d2fccfdfcdc36 --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md @@ -0,0 +1,222 @@ ++++ +title = "ClickHouse" +weight = 6 ++++ + +## Background Information + +ShardingSphere does not provide support for `driverClassName` of `com.clickhouse.jdbc.ClickHouseDriver` by default. +ShardingSphere's support for ClickHouse JDBC Driver is in the optional module. + +## Prerequisites + +To use a `jdbcUrl` like `jdbc:ch://localhost:8123/demo_ds_0` for the data node in the ShardingSphere configuration file, +the possible Maven dependencies are as follows, + +```xml + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-parser-sql-clickhouse + ${shardingsphere.version} + + + com.clickhouse + clickhouse-jdbc + http + 0.6.3 + + +``` + +## Configuration example + +### Start ClickHouse + +Write a Docker Compose file to start ClickHouse. + +```yaml +services: + clickhouse-server: + image: clickhouse/clickhouse-server:24.10.2.80 + ports: + - "8123:8123" +``` + +### Create business tables + +Use a third-party tool to create a business database and business table in ClickHouse. +Taking DBeaver Community as an example, if you use Ubuntu 22.04.4, you can quickly install it through Snapcraft. + +```shell +sudo apt update && sudo apt upgrade -y +sudo snap install dbeaver-ce +snap run dbeaver-ce +``` + +In DBeaver Community, use `jdbcUrl` of `jdbc:ch://localhost:8123/default`, `username` of `default` to connect to ClickHouse, +and leave `password` blank. +Execute the following SQL, + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +CREATE DATABASE demo_ds_0; +CREATE DATABASE demo_ds_1; +CREATE DATABASE demo_ds_2; +``` + +Use `jdbcUrl` of `jdbc:ch://localhost:8123/demo_ds_0`, +`jdbc:ch://localhost:8123/demo_ds_1` and `jdbc:ch://localhost:8123/demo_ds_2` +to connect to ClickHouse and execute the following SQL. + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +create table IF NOT EXISTS t_order ( + order_id Int64 NOT NULL DEFAULT rand(), + order_type Int32, + user_id Int32 NOT NULL, + address_id Int64 NOT NULL, + status String +) engine = MergeTree + primary key (order_id) + order by (order_id); + +TRUNCATE TABLE t_order; +``` + +### Create ShardingSphere data source in business project + +After the business project introduces the dependencies involved in `prerequisites`, +write the ShardingSphere data source configuration file `demo.yaml` on the classpath of the business project. + +```yaml +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_0 + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_1 + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_2 +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: INLINE + props: + algorithm-expression: ds_${user_id % 2} + keyGenerators: + snowflake: + type: SNOWFLAKE +``` + +### Enjoy integration + +Create a ShardingSphere data source to enjoy integration, + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +public class ExampleUtils { + void test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')"); + statement.executeQuery("SELECT * FROM t_order"); + statement.execute("alter table t_order delete where order_id=1"); + } + } +} +``` + +## Usage Limitations + +### SQL Limitations + +ShardingSphere JDBC DataSource does not yet support the execution of ClickHouse's `CREATE TABLE` statement and `TRUNCATE TABLE` statement. +Users should consider submitting a PR containing unit tests for ShardingSphere. + +### Key Generate restrictions + +The column type corresponding to the Key Generate function of ClickHouse itself is `UUID`, +and `UUID` is received as `java.util.UUID` in ClickHouse JDBC Driver, +refer to https://github.com/ClickHouse/ClickHouse/issues/56228 . +The column type corresponding to the Key Generate SPI implementation of ShardingSphere's `SNOWFLAKE` is `UInt64`, +which is received as `java.lang.Long` in ShardingSphere JDBC Driver. + +When configuring ShardingSphere to connect to ClickHouse, +if ShardingSphere is also configured to use the Key Generate SPI implementation of `SNOWFLAKE`, +the column type in the ClickHouse real database used by ShardingSphere's Key Generate function should not be set to `UUID`. + +Because `com.clickhouse.jdbc.ClickHouseConnection#prepareStatement(String, int)` of `com.clickhouse:clickhouse-jdbc:0.6.3:http` +Maven module intentionally throws an exception when `autoGeneratedKeys` is `java.sql.Statement.RETURN_GENERATED_KEYS`, +to prevent ShardingSphere from proxying `com.clickhouse.jdbc.internal.ClickHouseConnectionImpl` normally, +therefore, if users need to obtain the Key generated by ShardingSphere from the JDBC business code, +they need to set `autoGeneratedKeys` to `java.sql.Statement.NO_GENERATED_KEYS`. + +A possible example is as follows, + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.*; +public class ExampleTest { + long test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')", + Statement.NO_GENERATED_KEYS + )) { + preparedStatement.executeUpdate(); + try (ResultSet resultSet = preparedStatement.getGeneratedKeys()) { + if (resultSet.next()) { + return resultSet.getLong(1); + } + throw new RuntimeException(); + } + } + } +} +``` + +### Transaction Limitations + +ClickHouse does not support ShardingSphere integration-level local transactions, XA transactions, and Seata AT mode transactions. +More discussions are at https://github.com/ClickHouse/clickhouse-docs/issues/2300 . + +### Embedded ClickHouse Limitations + +The embedded ClickHouse `chDB` Java client has not been released yet. +ShardingSphere does not do integration testing for the SNAPSHOT version of https://github.com/chdb-io/chdb-java . +Refer to https://github.com/chdb-io/chdb/issues/243 . diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md index d071dbded284d..bb11ed7def68a 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md @@ -75,6 +75,10 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。 com.fasterxml.woodstox woodstox-core + + org.apache.commons + commons-text + diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md index 69e1c6d03893d..38c6781a568e2 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md @@ -77,6 +77,10 @@ The following is an example of a possible configuration, com.fasterxml.woodstox woodstox-core + + org.apache.commons + commons-text + diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md index 8ef2c84852667..64a17855097b9 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md @@ -101,3 +101,7 @@ public class ExampleUtils { } } ``` + +`org.testcontainers.jdbc.ContainerDatabaseDriver#killContainers()` +将立刻销毁所有由 `org.testcontainers.jdbc.ContainerDatabaseDriver` 创建的 Docker Container。 +默认情况下，通过 Junit 5 创建的单元测试是串行执行的，因此这一般不会造成问题。 diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md index f9faae7b4cd2b..72998c4a57ae3 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md @@ -100,3 +100,7 @@ public class ExampleUtils { } } ``` + +`org.testcontainers.jdbc.ContainerDatabaseDriver#killContainers()` +will immediately destroy all Docker Containers created by `org.testcontainers.jdbc.ContainerDatabaseDriver`. +By default, unit tests created by JUnit 5 are executed serially, so this is generally not a problem. diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json index 4522669fefb0c..31fa795126018 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json @@ -16,7 +16,7 @@ "name":"[Lcom.github.dockerjava.api.model.VolumesFrom;" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f428fce11f0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007fb05bce1828"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { @@ -63,6 +63,10 @@ "condition":{"typeReachable":"org.apache.shardingsphere.metadata.factory.ExternalMetaDataFactory"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.metadata.persist.service.version.MetaDataVersionPersistService"}, + "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" +}, { "condition":{"typeReachable":"org.apache.shardingsphere.mode.metadata.manager.SchemaMetaDataManager"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" @@ -696,11 +700,6 @@ "queryAllPublicConstructors":true, "methods":[{"name":"","parameterTypes":[] }, {"name":"add","parameterTypes":["long"] }, {"name":"sum","parameterTypes":[] }] }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.command.CommandExecutorTask"}, - "name":"java.util.concurrent.atomic.Striped64$Cell", - "fields":[{"name":"value"}] -}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.util.function.DoubleFunction", @@ -2079,7 +2078,7 @@ "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f428fb277d8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007fb05bb277d8"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber" }, { @@ -3608,10 +3607,5 @@ { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"}, "name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfigurationCustomizer" -}, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"}, - "name":"sun.security.provider.SecureRandom", - "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] } ] \ No newline at end of file diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json index fd5f3877576c5..6861bc941ff55 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json @@ -229,7 +229,7 @@ "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxyBackendHandlerFactory"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.checker.SQLExecutionChecker\\E" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.mysql.command.query.text.query.MySQLComQueryPacketExecutor"}, + "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.sql.hook.SQLExecutionHook\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.sql.prepare.AbstractExecutionPrepareEngine"}, diff --git a/test/native/pom.xml b/test/native/pom.xml index b8217ff5bfdf0..41a0d8be2796d 100644 --- a/test/native/pom.xml +++ b/test/native/pom.xml @@ -184,6 +184,10 @@ com.fasterxml.woodstox woodstox-core + + org.apache.commons + commons-text + diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java index 8a53808f55af6..c167304b9df0f 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java @@ -24,19 +24,18 @@ import org.apache.shardingsphere.test.natived.commons.repository.AddressRepository; import org.apache.shardingsphere.test.natived.commons.repository.OrderItemRepository; import org.apache.shardingsphere.test.natived.commons.repository.OrderRepository; -import org.awaitility.Awaitility; import javax.sql.DataSource; import java.sql.SQLException; import java.sql.Statement; -import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; +import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -63,24 +62,7 @@ public TestShardingService(final DataSource dataSource) { */ public void processSuccess() throws SQLException { final Collection orderIds = insertData(Statement.RETURN_GENERATED_KEYS); - Collection orders = orderRepository.selectAll(); - assertThat(orders.stream().map(Order::getOrderType).collect(Collectors.toList()), - equalTo(Arrays.asList(0, 0, 0, 0, 0, 1, 1, 1, 1, 1))); - assertThat(orders.stream().map(Order::getUserId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2, 4, 6, 8, 10, 1, 3, 5, 7, 9)))); - assertThat(orders.stream().map(Order::getAddressId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2L, 4L, 6L, 8L, 10L, 1L, 3L, 5L, 7L, 9L)))); - assertThat(orders.stream().map(Order::getStatus).collect(Collectors.toList()), - equalTo(IntStream.range(1, 11).mapToObj(i -> "INSERT_TEST").collect(Collectors.toList()))); - Collection orderItems = orderItemRepository.selectAll(); - assertThat(orderItems.stream().map(OrderItem::getUserId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2, 4, 6, 8, 10, 1, 3, 5, 7, 9)))); - assertThat(orderItems.stream().map(OrderItem::getPhone).collect(Collectors.toList()), - equalTo(IntStream.range(1, 11).mapToObj(i -> "13800000001").collect(Collectors.toList()))); - assertThat(orderItems.stream().map(OrderItem::getStatus).collect(Collectors.toList()), - equalTo(IntStream.range(1, 11).mapToObj(i -> "INSERT_TEST").collect(Collectors.toList()))); - assertThat(addressRepository.selectAll(), - equalTo(LongStream.range(1L, 11L).mapToObj(each -> new Address(each, "address_test_" + each)).collect(Collectors.toList()))); + extracted(); deleteData(orderIds); assertThat(orderRepository.selectAll(), equalTo(Collections.emptyList())); assertThat(orderItemRepository.selectAll(), equalTo(Collections.emptyList())); @@ -90,8 +72,6 @@ public void processSuccess() throws SQLException { /** * Process success in ClickHouse. - * TODO On low-performance devices like Github Actions, it takes longer to execute DELETE statements, - * which leads to the use of {@code org.awaitility.Awaitility.await()} here. Maybe there is room for improvement in Clickhouse JDBC Driver. * ClickHouse has not fully supported transactions. Refer to ClickHouse/clickhouse-docs#2300. * So ShardingSphere should not use {@link OrderItemRepository#assertRollbackWithTransactions()} in the method here. * @@ -99,29 +79,32 @@ public void processSuccess() throws SQLException { */ public void processSuccessInClickHouse() throws SQLException { final Collection orderIds = insertData(Statement.NO_GENERATED_KEYS); + extracted(); + deleteDataInClickHouse(orderIds); + assertThat(orderRepository.selectAll(), equalTo(Collections.emptyList())); + assertThat(orderItemRepository.selectAll(), equalTo(Collections.emptyList())); + assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList())); + } + + private void extracted() throws SQLException { Collection orders = orderRepository.selectAll(); assertThat(orders.stream().map(Order::getOrderType).collect(Collectors.toList()), - equalTo(Arrays.asList(0, 0, 0, 0, 0, 1, 1, 1, 1, 1))); - assertThat(orders.stream().map(Order::getUserId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2, 4, 6, 8, 10, 1, 3, 5, 7, 9)))); - assertThat(orders.stream().map(Order::getAddressId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2L, 4L, 6L, 8L, 10L, 1L, 3L, 5L, 7L, 9L)))); + equalTo(Stream.of(0, 0, 0, 0, 0, 1, 1, 1, 1, 1).collect(Collectors.toList()))); + assertThat(orders.stream().map(Order::getUserId).collect(Collectors.toSet()), + equalTo(Stream.of(2, 4, 6, 8, 10, 1, 3, 5, 7, 9).collect(Collectors.toSet()))); + assertThat(orders.stream().map(Order::getAddressId).collect(Collectors.toSet()), + equalTo(Stream.of(2L, 4L, 6L, 8L, 10L, 1L, 3L, 5L, 7L, 9L).collect(Collectors.toSet()))); assertThat(orders.stream().map(Order::getStatus).collect(Collectors.toList()), equalTo(IntStream.range(1, 11).mapToObj(i -> "INSERT_TEST").collect(Collectors.toList()))); Collection orderItems = orderItemRepository.selectAll(); - assertThat(orderItems.stream().map(OrderItem::getUserId).collect(Collectors.toList()), - equalTo(new ArrayList<>(Arrays.asList(2, 4, 6, 8, 10, 1, 3, 5, 7, 9)))); + assertThat(orderItems.stream().map(OrderItem::getUserId).collect(Collectors.toSet()), + equalTo(Stream.of(2, 4, 6, 8, 10, 1, 3, 5, 7, 9).collect(Collectors.toSet()))); assertThat(orderItems.stream().map(OrderItem::getPhone).collect(Collectors.toList()), equalTo(IntStream.range(1, 11).mapToObj(i -> "13800000001").collect(Collectors.toList()))); assertThat(orderItems.stream().map(OrderItem::getStatus).collect(Collectors.toList()), equalTo(IntStream.range(1, 11).mapToObj(i -> "INSERT_TEST").collect(Collectors.toList()))); - assertThat(addressRepository.selectAll(), - equalTo(LongStream.range(1L, 11L).mapToObj(each -> new Address(each, "address_test_" + each)).collect(Collectors.toList()))); - deleteDataInClickHouse(orderIds); - Awaitility.await().pollDelay(Duration.ofSeconds(5L)).until(() -> true); - assertThat(orderRepository.selectAll(), equalTo(Collections.emptyList())); - assertThat(orderItemRepository.selectAll(), equalTo(Collections.emptyList())); - assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList())); + assertThat(new HashSet<>(addressRepository.selectAll()), + equalTo(LongStream.range(1L, 11L).mapToObj(each -> new Address(each, "address_test_" + each)).collect(Collectors.toSet()))); } /** diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java index fab369ddf420b..2295102d790fd 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java @@ -41,7 +41,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; /** * Cannot use testcontainers-java style jdbcURL for Clickhouse Server due to unresolved @@ -84,7 +83,7 @@ void assertShardingInLocalTransactions() throws SQLException { jdbcUrlPrefix = "jdbc:ch://localhost:" + CONTAINER.getMappedPort(8123) + "/"; DataSource dataSource = createDataSource(); testShardingService = new TestShardingService(dataSource); - assertDoesNotThrow(() -> testShardingService.processSuccessInClickHouse()); + testShardingService.processSuccessInClickHouse(); } /**