Fixes the outdated description of Key Generation and Hadoop in HiveSe…

…rver2 integration doc (#33739)
apache · Nov 20, 2024 · 8722823 · 8722823
1 parent e344fc4
commit 8722823
Show file tree

Hide file tree

Showing 23 changed files with 98 additions and 188 deletions.
diff --git a/...ntent/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md b/...ntent/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md
@@ -40,11 +40,6 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。
         <artifactId>hive-service</artifactId>
         <version>4.0.1</version>
     </dependency>
-    <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-client-api</artifactId>
-        <version>3.3.6</version>
-    </dependency>
 </dependencies>
 ```
 
@@ -185,30 +180,30 @@ rules:
 
 ### 享受集成
 
-创建 ShardingSphere 的数据源，
+创建 ShardingSphere 的数据源以享受集成，
 
 ```java
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
-import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
 public class ExampleUtils {
-    DataSource createDataSource() {
+    void test() throws SQLException {
         HikariConfig config = new HikariConfig();
         config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml");
         config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
-        return new HikariDataSource(config);
+        try (HikariDataSource dataSource = new HikariDataSource(config);
+             Connection connection = dataSource.getConnection();
+             Statement statement = connection.createStatement()) {
+            statement.execute("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')");
+            statement.executeQuery("SELECT * FROM t_order");
+            statement.execute("DELETE FROM t_order WHERE order_id=1");
+        }
     }
 }
 ```
 
-可直接在此`javax.sql.DataSource`相关的 ShardingSphere DataSource 上执行逻辑 SQL，享受它，
-
-```sql
--- noinspection SqlNoDataSourceInspectionForFile
-INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, "INSERT_TEST")；
-DELETE FROM t_order WHERE order_id=1;
-```
-
 ## 使用限制
 
 ### 版本限制
@@ -258,16 +253,6 @@ ShardingSphere 当前对参数的解析仅支持以`jdbc:hive2://localhost:10000
 
 若用户需使用`;sess_var_list`或`#hive_var_list`的 jdbcURL 参数，考虑为 ShardingSphere 提交包含单元测试的 PR。
 
-
-### 分布式序列限制
-
-由于 `org.apache.hive.jdbc.HiveStatement` 未实现 `java.sql.Statement#getGeneratedKeys()`，
-ShardingSphere JDBC Connection 无法通过 `java.sql.Statement.RETURN_GENERATED_KEYS` 获得 ShardingSphere 生成的雪花 ID 等分布式序列。
-
-若用户需要通过 `java.sql.Statement.RETURN_GENERATED_KEYS` 从 HiveServer2 获得 ShardingSphere 生成的雪花 ID 等分布式序列，
-用户应当考虑在 Hive 的主分支实现 `java.sql.DatabaseMetaData#getURL()`，
-而不是尝试修改 ShardingSphere 的内部类。
-
 ### 在 ShardingSphere 数据源上使用 DML SQL 语句的前提条件
 
 为了能够使用 `delete` 等 DML SQL 语句，当连接到 HiveServer2 时，用户应当考虑在 ShardingSphere JDBC 中仅使用支持 ACID 的表。

diff --git a/...ntent/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md b/...ntent/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md
@@ -41,11 +41,6 @@ The possible Maven dependencies are as follows.
         <artifactId>hive-service</artifactId>
         <version>4.0.1</version>
     </dependency>
-    <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-client-api</artifactId>
-        <version>3.3.6</version>
-    </dependency>
 </dependencies>
 ```
 
@@ -188,30 +183,30 @@ rules:
 
 ### Enjoy the integration
 
-Create a ShardingSphere data source,
+Create a ShardingSphere data source to enjoy the integration.
 
 ```java
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
-import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
 public class ExampleUtils {
-    DataSource createDataSource() {
+    void test() throws SQLException {
         HikariConfig config = new HikariConfig();
         config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml");
         config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
-        return new HikariDataSource(config);
+        try (HikariDataSource dataSource = new HikariDataSource(config);
+             Connection connection = dataSource.getConnection();
+             Statement statement = connection.createStatement()) {
+            statement.execute("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, 'INSERT_TEST')");
+            statement.executeQuery("SELECT * FROM t_order");
+            statement.execute("DELETE FROM t_order WHERE order_id=1");
+        }
     }
 }
 ```
 
-You can directly execute logical SQL on the ShardingSphere DataSource related to this `javax.sql.DataSource`, enjoy it,
-
-```sql
--- noinspection SqlNoDataSourceInspectionForFile
-INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (1, 1, 1, "INSERT_TEST")；
-DELETE FROM t_order WHERE order_id=1;
-```
-
 ## Usage Restrictions
 
 ### Version Restrictions
@@ -265,15 +260,6 @@ ShardingSphere currently only supports the `;hive_conf_list` part represented by
 If users need to use the jdbcURL parameters of `;sess_var_list` or `#hive_var_list`, 
 consider submitting a PR containing unit tests for ShardingSphere.
 
-### Distributed sequence limitations
-
-Since `org.apache.hive.jdbc.HiveStatement` does not implement `java.sql.Statement#getGeneratedKeys()`,
-ShardingSphere JDBC Connection cannot obtain distributed sequences such as Snowflake ID generated by ShardingSphere through `java.sql.Statement.RETURN_GENERATED_KEYS`.
-
-If users need to obtain distributed sequences such as Snowflake ID generated by ShardingSphere from HiveServer2 through `java.sql.Statement.RETURN_GENERATED_KEYS`,
-users should consider implementing `java.sql.DatabaseMetaData#getURL()` in the main branch of Hive,
-rather than trying to modify the internal classes of ShardingSphere.
-
 ### Prerequisites for using DML SQL statements on ShardingSphere data sources
 
 In order to be able to use DML SQL statements such as `delete`, 

diff --git a/...ative-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/...ative-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json
@@ -71,10 +71,6 @@
   "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"},
   "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
 },
-{
-  "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.jdbc.datasource.JDBCBackendDataSource"},
-  "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
-},
 {
   "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.postgresql.command.query.extended.Portal"},
   "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
@@ -696,6 +692,11 @@
   "queryAllPublicConstructors":true,
   "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"add","parameterTypes":["long"] }, {"name":"sum","parameterTypes":[] }]
 },
+{
+  "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.command.CommandExecutorTask"},
+  "name":"java.util.concurrent.atomic.Striped64$Cell",
+  "fields":[{"name":"value"}]
+},
 {
   "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"},
   "name":"java.util.function.DoubleFunction",
@@ -2074,7 +2075,7 @@
   "queryAllDeclaredMethods":true
 },
 {
-  "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f619fb277d8"},
+  "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f401fb273a8"},
   "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber"
 },
 {

diff --git a/...ive/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java b/...ive/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java
@@ -128,16 +128,16 @@ public void processSuccessInClickHouse() throws SQLException {
      * Process success in Hive.
      * Hive has not fully supported BEGIN, COMMIT, and ROLLBACK. Refer to <a href="https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions">Hive Transactions</a>.
      * So ShardingSphere should not use {@link OrderItemRepository#assertRollbackWithTransactions()}
-     * TODO It looks like HiveServer2 insert and delete statements are inserted out of order. Waiting for further investigation.
+     * TODO It looks like HiveServer2 insert statements are inserted out of order. Waiting for further investigation.
      *  The result of the insert is not currently asserted.
-     * TODO It is currently not convenient to operate on the `t_order` and `t_order_item` tables because
-     *  {@link org.apache.hive.jdbc.HiveStatement} does not implement {@link org.apache.hive.jdbc.HiveStatement#getGeneratedKeys()}
      *
      * @throws SQLException An exception that provides information on a database access error or other errors.
      */
     public void processSuccessInHive() throws SQLException {
-        insertDataInHive();
-        deleteDataInHive();
+        final Collection<Long> orderIds = insertData(Statement.RETURN_GENERATED_KEYS);
+        deleteData(orderIds);
+        assertThat(orderRepository.selectAll(), equalTo(Collections.emptyList()));
+        assertThat(orderItemRepository.selectAll(), equalTo(Collections.emptyList()));
         assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList()));
     }
 
@@ -173,30 +173,6 @@ public Collection<Long> insertData(final int autoGeneratedKeys) throws SQLExcept
         return result;
     }
 
-    /**
-     * Insert data in Hive.
-     * {@link org.apache.hive.jdbc.HiveStatement} does not implement {@link java.sql.Statement#getGeneratedKeys()},
-     * so the snowflake ID generated by ShardingSphere cannot be obtained.
-     */
-    public void insertDataInHive() {
-        IntStream.range(1, 11).forEach(this::insertSingleDataInHive);
-    }
-
-    private void insertSingleDataInHive(final int action) {
-        Order order = new Order();
-        order.setUserId(action);
-        order.setOrderType(action % 2);
-        order.setAddressId(action);
-        order.setStatus("INSERT_TEST");
-        Address address = new Address((long) action, "address_test_" + action);
-        try {
-            orderRepository.insertInHive(order);
-            addressRepository.insert(address);
-        } catch (final SQLException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
     /**
      * Delete data.
      *
@@ -227,20 +203,6 @@ public void deleteDataInClickHouse(final Collection<Long> orderIds) throws SQLEx
         }
     }
 
-    /**
-     * Delete data in Hive.
-     */
-    public void deleteDataInHive() {
-        LongStream.range(1, 11).forEach(action -> {
-            try {
-                orderRepository.delete(action);
-                addressRepository.delete(action);
-            } catch (final SQLException exception) {
-                throw new RuntimeException(exception);
-            }
-        });
-    }
-
     /**
      * Clean environment.
      *

diff --git a/...t/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java b/...t/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java
@@ -243,6 +243,7 @@ public Long insert(final OrderItem orderItem) throws SQLException {
      * @return orderItemId of the insert statement
      * @throws SQLException SQL exception
      */
+    @SuppressWarnings("MagicConstant")
     public Long insert(final OrderItem orderItem, final int autoGeneratedKeys) throws SQLException {
         String sql = "INSERT INTO t_order_item (order_id, user_id, phone, status) VALUES (?, ?, ?, ?)";
         try (

diff --git a/.../test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java b/.../test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java
@@ -289,27 +289,6 @@ public Long insert(final Order order, final int autoGeneratedKeys) throws SQLExc
         return order.getOrderId();
     }
 
-    /**
-     * insert Order to table in HiveServer2.
-     * {@link org.apache.hive.jdbc.HiveStatement} does not implement {@link java.sql.Statement#getGeneratedKeys()},
-     * so the snowflake ID generated by ShardingSphere cannot be obtained.
-     *
-     * @param order             order
-     * @throws SQLException SQL Exception
-     */
-    public void insertInHive(final Order order) throws SQLException {
-        String sql = "INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (?, ?, ?, ?)";
-        try (
-                Connection connection = dataSource.getConnection();
-                PreparedStatement preparedStatement = connection.prepareStatement(sql, Statement.NO_GENERATED_KEYS)) {
-            preparedStatement.setInt(1, order.getUserId());
-            preparedStatement.setInt(2, order.getOrderType());
-            preparedStatement.setLong(3, order.getAddressId());
-            preparedStatement.setString(4, order.getStatus());
-            preparedStatement.executeUpdate();
-        }
-    }
-
     /**
      * delete by orderId.
      *

diff --git a/...rc/test/java/org/apache/shardingsphere/test/natived/jdbc/transactions/base/SeataTest.java b/...rc/test/java/org/apache/shardingsphere/test/natived/jdbc/transactions/base/SeataTest.java
@@ -21,7 +21,6 @@
 import com.zaxxer.hikari.HikariDataSource;
 import org.apache.http.HttpStatus;
 import org.apache.shardingsphere.test.natived.commons.TestShardingService;
-import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -33,7 +32,6 @@
 
 import javax.sql.DataSource;
 import java.sql.SQLException;
-import java.time.Duration;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
@@ -68,12 +66,10 @@ static void afterAll() {
 
     /**
      * TODO Just checking `/health` of `7091` port of Seata Server is not enough to indicate that Seata Server is ready.
-     *  The use of {@code Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(() -> true);} needs to be removed.
      * @throws SQLException An exception that provides information on a database access error or other errors.
      */
     @Test
     void assertShardingInSeataTransactions() throws SQLException {
-        Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(() -> true);
         DataSource dataSource = createDataSource(CONTAINER.getMappedPort(8091));
         testShardingService = new TestShardingService(dataSource);
         initEnvironment();

diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml
@@ -44,12 +44,12 @@ rules:
   - !SHARDING
     tables:
       t_order:
-        actualDataNodes:
+        actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
         keyGenerateStrategy:
           column: order_id
           keyGeneratorName: snowflake
       t_order_item:
-        actualDataNodes:
+        actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item
         keyGenerateStrategy:
           column: order_item_id
           keyGeneratorName: snowflake

diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive.yaml
@@ -38,12 +38,12 @@ rules:
 - !SHARDING
   tables:
     t_order:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
       keyGenerateStrategy:
         column: order_id
         keyGeneratorName: snowflake
     t_order_item:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item
       keyGenerateStrategy:
         column: order_item_id
         keyGeneratorName: snowflake

diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/mysql.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/mysql.yaml
@@ -44,12 +44,12 @@ rules:
 - !SHARDING
   tables:
     t_order:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
       keyGenerateStrategy:
         column: order_id
         keyGeneratorName: snowflake
     t_order_item:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item
       keyGenerateStrategy:
         column: order_item_id
         keyGeneratorName: snowflake

diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/opengauss.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/opengauss.yaml
@@ -44,12 +44,12 @@ rules:
   - !SHARDING
     tables:
       t_order:
-        actualDataNodes:
+        actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
         keyGenerateStrategy:
           column: order_id
           keyGeneratorName: snowflake
       t_order_item:
-        actualDataNodes:
+        actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item
         keyGenerateStrategy:
           column: order_item_id
           keyGeneratorName: snowflake

diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/postgresql.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/postgresql.yaml
@@ -38,12 +38,12 @@ rules:
 - !SHARDING
   tables:
     t_order:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
       keyGenerateStrategy:
         column: order_id
         keyGeneratorName: snowflake
     t_order_item:
-      actualDataNodes:
+      actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item, ds_2.t_order_item
       keyGenerateStrategy:
         column: order_item_id
         keyGeneratorName: snowflake