Merge pull request #596 from zhicwu/roaringbitmap

RoaringBitmap support
ClickHouse · Mar 23, 2021 · 9267440 · 9267440
2 parents a90da54 + bbfca81
commit 9267440
Show file tree

Hide file tree

Showing 34 changed files with 1,509 additions and 305 deletions.
diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml
@@ -61,6 +61,7 @@ jobs:
           # added,diff_context,file,nofilter
           filter_mode: 'added'
         if: github.event_name == 'pull_request_target' || github.event.inputs.pr != ''
+        continue-on-error: true
       - name: Update sonar config
         run: |
           sed -i -e 's|^\(.*<sonar.projectKey>\).*\(</sonar.projectKey>\)$|\1ClickHouse_clickhouse-jdbc\2|' \
@@ -73,3 +74,4 @@ jobs:
         run: |
           find . -type f -name "log4j.*" -exec rm -fv '{}' \;
           mvn -q --batch-mode -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
+        continue-on-error: true
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -34,7 +34,8 @@ jobs:
       matrix:
         java: [8, 11, 15]
         # most recent LTS releases as well as latest stable builds
-        clickhouse: ["20.8", "20.10", "20.12", "21.2", "latest"]
+        clickhouse: ["20.8", "21.3", "latest"]
+      fail-fast: false
     name: Build using JDK ${{ matrix.java }} against ClickHouse ${{ matrix.clickhouse }}
     steps:
       - name: Check out Git repository

diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml
@@ -1,4 +1,4 @@
-name: TimeZone Test
+name: TimeZone
 
 on:
   push:

diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml
@@ -18,10 +18,9 @@
 
     <properties>
         <clickhouse4j-driver.version>1.4.4</clickhouse4j-driver.version>
-        <native-driver.version>2.5.3</native-driver.version>
         <mariadb-driver.version>2.7.2</mariadb-driver.version>
         <mysql-driver.version>8.0.23</mysql-driver.version>
-        <native-driver.version>2.5.3</native-driver.version>
+        <native-driver.version>2.5.4</native-driver.version>
         <testcontainers.version>1.15.2</testcontainers.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <jmh.version>1.27</jmh.version>

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java
@@ -4,7 +4,6 @@
 import java.sql.Statement;
 import java.util.Collections;
 import java.util.Random;
-
 import org.openjdk.jmh.annotations.Benchmark;
 
 public class Basic extends JdbcBenchmark {

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java
@@ -4,7 +4,6 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
-
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java
@@ -1,5 +1,8 @@
 package tech.clickhouse.benchmark;
 
+/**
+ * Constant interface.
+ */
 public interface Constants {
     public static final String CLICKHOUSE_DRIVER = "clickhouse-jdbc";
 

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java
@@ -4,7 +4,6 @@
 // import java.util.Collections;
 import java.util.Enumeration;
 import java.util.Random;
-
 import org.openjdk.jmh.annotations.Benchmark;
 
 public class Insertion extends JdbcBenchmark {

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java
@@ -1,15 +1,25 @@
 package tech.clickhouse.benchmark;
 
-import org.openjdk.jmh.annotations.*;
-
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Enumeration;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Base class for JDBC driver benchmarking.
+ */
 @State(Scope.Benchmark)
 @Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)
 @Measurement(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java
@@ -16,12 +16,14 @@ public enum JdbcDriver {
 
     // MariaDB Java Client
     MariadbJavaClient("org.mariadb.jdbc.Driver",
-            "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
+            "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false"
+                    + "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
             Constants.MYSQL_PORT),
 
     // MySQL Connector/J
     MysqlConnectorJava("com.mysql.cj.jdbc.Driver",
-            "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC",
+            "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false"
+                    + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC",
             Constants.MYSQL_PORT);
 
     private final String className;

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java
@@ -4,7 +4,6 @@
 import java.sql.Statement;
 import java.sql.Timestamp;
 import java.util.Random;
-
 import org.openjdk.jmh.annotations.Benchmark;
 
 public class Query extends JdbcBenchmark {

diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java
@@ -1,12 +1,13 @@
 package tech.clickhouse.benchmark;
 
+import static java.time.temporal.ChronoUnit.SECONDS;
+
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.InterfaceAddress;
 import java.net.NetworkInterface;
 import java.time.Duration;
 import java.util.Enumeration;
-
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
@@ -16,8 +17,6 @@
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.images.builder.ImageFromDockerfile;
 
-import static java.time.temporal.ChronoUnit.SECONDS;
-
 @State(Scope.Benchmark)
 public class ServerState {
     static String getLocalIpAddress() {

diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml
@@ -72,6 +72,10 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.RoaringBitmap</groupId>
+            <artifactId>RoaringBitmap</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -198,6 +202,10 @@
                             <pattern>net.jpountz</pattern>
                             <shadedPattern>${shade.base}.jpountz</shadedPattern>
                         </relocation>
+                        <relocation>
+                            <pattern>org.roaringbitmap</pattern>
+                            <shadedPattern>${shade.base}.bitmap</shadedPattern>
+                        </relocation>
                         <relocation>
                             <pattern>org.slf4j</pattern>
                             <shadedPattern>${shade.base}.slf4j</shadedPattern>

diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java
@@ -19,6 +19,7 @@ public ByteFragment(byte[] buf, int start, int len) {
     }
 
     public static ByteFragment fromString(String str) {
+        // https://bugs.openjdk.java.net/browse/JDK-6219899
         byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
         return new ByteFragment(bytes, 0, bytes.length);
     }

diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java
@@ -1,9 +1,11 @@
 package ru.yandex.clickhouse.response;
 
 import java.util.TimeZone;
-
 import ru.yandex.clickhouse.domain.ClickHouseDataType;
 
+/**
+ * This class represents a column defined in database.
+ */
 public final class ClickHouseColumnInfo {
 
     private static final String KEYWORD_NULLABLE = "Nullable";
@@ -22,12 +24,21 @@ public final class ClickHouseColumnInfo {
     private int scale;
     private ClickHouseColumnInfo keyInfo;
     private ClickHouseColumnInfo valueInfo;
+    private String functionName;
 
     @Deprecated
     public static ClickHouseColumnInfo parse(String typeInfo, String columnName) {
         return parse(typeInfo, columnName, null);
     }
 
+    /**
+     * Parse given type string.
+     *
+     * @param typeInfo type defined in database
+     * @param columnName column name
+     * @param serverTimeZone server time zone
+     * @return parsed type
+     */
     public static ClickHouseColumnInfo parse(String typeInfo, String columnName, TimeZone serverTimeZone) {
         ClickHouseColumnInfo column = new ClickHouseColumnInfo(typeInfo, columnName);
         int currIdx = 0;
@@ -61,68 +72,74 @@ public static ClickHouseColumnInfo parse(String typeInfo, String columnName, Tim
         column.scale = dataType.getDefaultScale();
         column.timeZone = serverTimeZone;
         currIdx = endIdx;
-        if (endIdx == typeInfo.length()
-            || !typeInfo.startsWith("(", currIdx))
-        {
+        if (endIdx == typeInfo.length() || !typeInfo.startsWith("(", currIdx)) {
             return column;
         }
 
         switch (dataType) {
-            case DateTime :
-                String[] argsDT = splitArgs(typeInfo, currIdx);
-                if (argsDT.length == 2) { // same as DateTime64
-                    column.scale = Integer.parseInt(argsDT[0]);
-                    column.timeZone = TimeZone.getTimeZone(argsDT[1].replace("'", ""));
-                } else if (argsDT.length == 1) { // same as DateTime32
-                    // unfortunately this will fall back to GMT if the time zone
-                    // cannot be resolved
-                    TimeZone tz = TimeZone.getTimeZone(argsDT[0].replace("'", ""));
-                    column.timeZone = tz;
-                }
-                break;
-            case DateTime32:
-                String[] argsD32 = splitArgs(typeInfo, currIdx);
-                if (argsD32.length == 1) {
-                    // unfortunately this will fall back to GMT if the time zone
-                    // cannot be resolved
-                    TimeZone tz = TimeZone.getTimeZone(argsD32[0].replace("'", ""));
-                    column.timeZone = tz;
-                }
-                break;
-            case DateTime64:
-                String[] argsD64 = splitArgs(typeInfo, currIdx);
-                if (argsD64.length == 2) {
-                    column.scale = Integer.parseInt(argsD64[0]);
-                    column.timeZone = TimeZone.getTimeZone(argsD64[1].replace("'", ""));
-                }
-                break;
-            case Decimal :
-                String[] argsDecimal = splitArgs(typeInfo, currIdx);
-                if (argsDecimal.length == 2) {
-                    column.precision = Integer.parseInt(argsDecimal[0]);
-                    column.scale = Integer.parseInt(argsDecimal[1]);
-                }
-                break;
-            case Decimal32 :
-            case Decimal64 :
-            case Decimal128 :
-            case Decimal256 :
-                String[] argsScale = splitArgs(typeInfo, currIdx);
-                column.scale = Integer.parseInt(argsScale[0]);
-                break;
-            case FixedString :
-                String[] argsPrecision = splitArgs(typeInfo, currIdx);
-                column.precision = Integer.parseInt(argsPrecision[0]);
-                break;
-            case Map:
-                String[] argsMap = splitArgs(typeInfo, currIdx);
-                if (argsMap.length == 2) {
-                    column.keyInfo = ClickHouseColumnInfo.parse(argsMap[0], columnName + "Key", serverTimeZone);
-                    column.valueInfo = ClickHouseColumnInfo.parse(argsMap[1], columnName + "Value", serverTimeZone);
-                }
-                break;
-            default :
-                break;
+        case AggregateFunction :
+            String[] argsAf = splitArgs(typeInfo, currIdx);
+            column.functionName = argsAf[0];
+            column.arrayBaseType = ClickHouseDataType.Unknown;
+            if (argsAf.length == 2) {
+                column.arrayBaseType = ClickHouseDataType.fromTypeString(argsAf[1]);
+            }
+            break;
+        case DateTime :
+            String[] argsDt = splitArgs(typeInfo, currIdx);
+            if (argsDt.length == 2) { // same as DateTime64
+                column.scale = Integer.parseInt(argsDt[0]);
+                column.timeZone = TimeZone.getTimeZone(argsDt[1].replace("'", ""));
+            } else if (argsDt.length == 1) { // same as DateTime32
+                // unfortunately this will fall back to GMT if the time zone
+                // cannot be resolved
+                TimeZone tz = TimeZone.getTimeZone(argsDt[0].replace("'", ""));
+                column.timeZone = tz;
+            }
+            break;
+        case DateTime32:
+            String[] argsD32 = splitArgs(typeInfo, currIdx);
+            if (argsD32.length == 1) {
+                // unfortunately this will fall back to GMT if the time zone
+                // cannot be resolved
+                TimeZone tz = TimeZone.getTimeZone(argsD32[0].replace("'", ""));
+                column.timeZone = tz;
+            }
+            break;
+        case DateTime64:
+            String[] argsD64 = splitArgs(typeInfo, currIdx);
+            if (argsD64.length == 2) {
+                column.scale = Integer.parseInt(argsD64[0]);
+                column.timeZone = TimeZone.getTimeZone(argsD64[1].replace("'", ""));
+            }
+            break;
+        case Decimal :
+            String[] argsDecimal = splitArgs(typeInfo, currIdx);
+            if (argsDecimal.length == 2) {
+                column.precision = Integer.parseInt(argsDecimal[0]);
+                column.scale = Integer.parseInt(argsDecimal[1]);
+            }
+            break;
+        case Decimal32 :
+        case Decimal64 :
+        case Decimal128 :
+        case Decimal256 :
+            String[] argsScale = splitArgs(typeInfo, currIdx);
+            column.scale = Integer.parseInt(argsScale[0]);
+            break;
+        case FixedString :
+            String[] argsPrecision = splitArgs(typeInfo, currIdx);
+            column.precision = Integer.parseInt(argsPrecision[0]);
+            break;
+        case Map:
+            String[] argsMap = splitArgs(typeInfo, currIdx);
+            if (argsMap.length == 2) {
+                column.keyInfo = ClickHouseColumnInfo.parse(argsMap[0], columnName + "Key", serverTimeZone);
+                column.valueInfo = ClickHouseColumnInfo.parse(argsMap[1], columnName + "Value", serverTimeZone);
+            }
+            break;
+        default:
+            break;
         }
 
         return column;
@@ -153,8 +170,9 @@ public String getOriginalTypeName() {
     }
 
     /**
-     * @return the type name returned from the database, without modifiers, i.e.
-     *         Nullable or LowCardinality
+     * Get the type name returned from the database, without modifiers, i.e. Nullable or LowCardinality.
+     *
+     * @return the type name returned from the database
      */
     public String getCleanTypeName() {
         if (!nullable && !lowCardinality) {
@@ -226,4 +244,8 @@ public ClickHouseColumnInfo getKeyInfo() {
     public ClickHouseColumnInfo getValueInfo() {
         return this.valueInfo;
     }
+
+    public String getFunctionName() {
+        return this.functionName;
+    }
 }