Migrate to Testcontainers (#297)

* add testcontainers deps * migrate to testcontainers * remove docker-compose stuffs * remvoe duplicated dependencies
housepower · Jan 25, 2021 · a0444d2 · a0444d2
1 parent cad3e23
commit a0444d2
Show file tree

Hide file tree

Showing 15 changed files with 158 additions and 81 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -21,8 +21,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 8
-      - name: Start clickhouse-server
-        run: docker-compose up -d
       - run: sed -i 's/TRACE/INFO/g' clickhouse-native-jdbc/src/test/resources/simplelogger.properties
       - run: sed -i 's/DEBUG/INFO/g' clickhouse-native-jdbc/src/test/resources/simplelogger.properties
 

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -24,8 +24,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      - name: Start clickhouse-server
-        run: docker-compose up -d
       - name: Run integration tests with Scala 2.11 & Spark 2.4.7
         run: mvn -B clean verify
       - name: upload code coverage report

diff --git a/clickhouse-integration/clickhouse-integration-spark/pom.xml b/clickhouse-integration/clickhouse-integration-spark/pom.xml
@@ -60,11 +60,27 @@
             <artifactId>clickhouse-native-jdbc-shaded</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>ru.yandex.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <classifier>shaded</classifier>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>clickhouse</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/clickhouse-native-jdbc/pom.xml b/clickhouse-native-jdbc/pom.xml
@@ -82,6 +82,16 @@
             <artifactId>guava</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>clickhouse</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...src/main/java/com/github/housepower/jdbc/statement/ClickHousePreparedInsertStatement.java b/...src/main/java/com/github/housepower/jdbc/statement/ClickHousePreparedInsertStatement.java
@@ -284,7 +284,7 @@ private Object convertToCkDataType(IDataType<?, ?> type, Object obj) throws Clic
             }
             return ((ClickHouseStruct) obj).mapAttributes(((DataTypeTuple) type).getNestedTypes(), unchecked(this::convertToCkDataType));
         }
-        LOG.debug("unhandled type: {}", obj.getClass());
+        LOG.debug("unhandled type: {}[{}]", type.name(), obj.getClass());
         return obj;
     }
 }
diff --git a/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/AbstractITest.java b/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/AbstractITest.java
@@ -15,6 +15,10 @@
 package com.github.housepower.jdbc;
 
 import com.github.housepower.misc.StrUtil;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.ClickHouseContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
 
 import java.io.Serializable;
 import java.sql.Connection;
@@ -26,15 +30,35 @@
 
 import javax.sql.DataSource;
 
+@Testcontainers
 public abstract class AbstractITest implements Serializable {
 
     protected static final ZoneId CLIENT_TZ = ZoneId.systemDefault();
     protected static final ZoneId SERVER_TZ = ZoneId.of("UTC");
     protected static final String DRIVER_CLASS_NAME = "com.github.housepower.jdbc.ClickHouseDriver";
-    protected static final int SERVER_PORT = Integer.parseInt(System.getProperty("CLICK_HOUSE_SERVER_PORT", "9000"));
-    protected static final String SERVER_USER = System.getProperty("CLICK_HOUSE_SERVER_USER", "default");
-    protected static final String SERVER_PASSWORD = System.getProperty("CLICK_HOUSE_SERVER_PASSWORD", "");
-    protected static final String SERVER_DATABASE = System.getProperty("CLICK_HOUSE_SERVER_DATABASE");
+
+    public static final String CLICKHOUSE_IMAGE = System.getProperty("CLICKHOUSE_IMAGE", "yandex/clickhouse-server:20.8");
+
+    protected static final String CLICKHOUSE_USER = System.getProperty("CLICKHOUSE_USER", "default");
+    protected static final String CLICKHOUSE_PASSWORD = System.getProperty("CLICKHOUSE_PASSWORD", "");
+    protected static final String CLICKHOUSE_DB = System.getProperty("CLICKHOUSE_DB", "");
+
+    @Container
+    public static ClickHouseContainer container = (ClickHouseContainer) new ClickHouseContainer(CLICKHOUSE_IMAGE)
+            .withEnv("CLICKHOUSE_USER", CLICKHOUSE_USER)
+            .withEnv("CLICKHOUSE_PASSWORD", CLICKHOUSE_PASSWORD)
+            .withEnv("CLICKHOUSE_DB", CLICKHOUSE_DB);
+
+    protected static String CK_HOST;
+    protected static String CK_IP;
+    protected static int CK_PORT;
+
+    @BeforeAll
+    public static void extractContainerInfo() {
+        CK_HOST = container.getHost();
+        CK_IP = container.getContainerIpAddress();
+        CK_PORT = container.getMappedPort(ClickHouseContainer.NATIVE_PORT);
+    }
 
     /**
      * just for compatible with scala
@@ -45,29 +69,26 @@ protected String getJdbcUrl() {
 
     protected String getJdbcUrl(Object... params) {
         StringBuilder sb = new StringBuilder();
-        sb.append("jdbc:clickhouse://127.0.0.1:").append(SERVER_PORT);
-        if (SERVER_DATABASE != null) {
-            sb.append("/").append(SERVER_DATABASE);
+        int port = container.getMappedPort(ClickHouseContainer.NATIVE_PORT);
+        sb.append("jdbc:clickhouse://").append(container.getHost()).append(":").append(port);
+        if (StrUtil.isNotEmpty(CLICKHOUSE_DB)) {
+            sb.append("/").append(container.getDatabaseName());
         }
         for (int i = 0; i + 1 < params.length; i++) {
-            if (i == 0) {
-                sb.append("?");
-            } else {
-                sb.append("&");
-            }
+            sb.append(i == 0 ? "?" : "&");
             sb.append(params[i]).append("=").append(params[i + 1]);
         }
-        
+
         // Add user
         sb.append(params.length < 2 ? "?" : "&");
-        sb.append("user=").append(SERVER_USER);
-        
+        sb.append("user=").append(container.getUsername());
+
         // Add password
-        // Currently we ignore the blan password
-        if (!StrUtil.isBlank(SERVER_PASSWORD)) {
-            sb.append("&password=").append(SERVER_PASSWORD);
+        // ignore blank password
+        if (!StrUtil.isBlank(CLICKHOUSE_PASSWORD)) {
+            sb.append("&password=").append(container.getPassword());
         }
-   
+
         return sb.toString();
     }
 

diff --git a/...tive-jdbc/src/test/java/com/github/housepower/jdbc/BalancedClickhouseDataSourceITest.java b/...tive-jdbc/src/test/java/com/github/housepower/jdbc/BalancedClickhouseDataSourceITest.java
@@ -22,24 +22,25 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.time.Duration;
+import java.util.Locale;
 import java.util.Properties;
 
 import static org.junit.jupiter.api.Assertions.*;
 
-public class BalancedClickhouseDataSourceITest {
+public class BalancedClickhouseDataSourceITest extends AbstractITest {
 
-    private static BalancedClickhouseDataSource dataSource;
-    private static BalancedClickhouseDataSource doubleDataSource;
+    private static BalancedClickhouseDataSource singleDs;
+    private static BalancedClickhouseDataSource dualDs;
 
     @BeforeEach
     public void reset() {
-        dataSource = new BalancedClickhouseDataSource("jdbc:clickhouse://127.0.0.1:9000");
-        doubleDataSource = new BalancedClickhouseDataSource("jdbc:clickhouse://127.0.0.1:9000,127.0.0.1:9000");
+        singleDs = new BalancedClickhouseDataSource(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s", CK_HOST, CK_PORT));
+        dualDs = new BalancedClickhouseDataSource(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", CK_HOST, CK_PORT, CK_IP, CK_PORT));
     }
 
     @Test
     public void testSingleDatabaseConnection() throws Exception {
-        Connection connection = dataSource.getConnection();
+        Connection connection = singleDs.getConnection();
         connection.createStatement().execute("CREATE DATABASE IF NOT EXISTS test");
         connection.createStatement().execute("DROP TABLE IF EXISTS test.insert_test");
         connection.createStatement().execute(
@@ -58,14 +59,14 @@ public void testSingleDatabaseConnection() throws Exception {
 
     @Test
     public void testDoubleDatabaseConnection() throws Exception {
-        Connection connection = doubleDataSource.getConnection();
+        Connection connection = dualDs.getConnection();
         connection.createStatement().execute("CREATE DATABASE IF NOT EXISTS test");
-        connection = doubleDataSource.getConnection();
+        connection = dualDs.getConnection();
         connection.createStatement().execute("DROP TABLE IF EXISTS test.insert_test");
         connection.createStatement().execute(
                 "CREATE TABLE IF NOT EXISTS test.insert_test (i Int32, s String) ENGINE = TinyLog");
 
-        connection = doubleDataSource.getConnection();
+        connection = dualDs.getConnection();
 
         PreparedStatement statement = connection.prepareStatement("INSERT INTO test.insert_test (s, i) VALUES (?, ?)");
         statement.setString(1, "asd");
@@ -78,7 +79,7 @@ public void testDoubleDatabaseConnection() throws Exception {
         assertEquals("asd", rs.getString("s"));
         assertEquals(42, rs.getInt("i"));
 
-        connection = doubleDataSource.getConnection();
+        connection = dualDs.getConnection();
 
         statement = connection.prepareStatement("INSERT INTO test.insert_test (s, i) VALUES (?, ?)");
         statement.setString(1, "asd");
@@ -96,8 +97,8 @@ public void testDoubleDatabaseConnection() throws Exception {
 
     @Test
     public void testCorrectActualizationDatabaseConnection() throws Exception {
-        dataSource.actualize();
-        try (Connection connection = dataSource.getConnection()) {
+        singleDs.actualize();
+        try (Connection connection = singleDs.getConnection()) {
             assertTrue(connection.isValid(1000));
         }
     }
@@ -106,7 +107,7 @@ public void testCorrectActualizationDatabaseConnection() throws Exception {
     @Test
     public void testDisableConnection() {
         BalancedClickhouseDataSource badDatasource = new BalancedClickhouseDataSource(
-                "jdbc:clickhouse://not.existed.url:9000", new Properties());
+                "jdbc:clickhouse://not.existed.url:" + CK_PORT, new Properties());
         badDatasource.actualize();
         try {
             Connection connection = badDatasource.getConnection();
@@ -120,7 +121,7 @@ public void testDisableConnection() {
     @Test
     public void testWorkWithEnabledUrl() throws Exception {
         BalancedClickhouseDataSource halfDatasource = new BalancedClickhouseDataSource(
-                "jdbc:clickhouse://not.existed.url:9000,127.0.0.1:9000", new Properties());
+                String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", "not.existed.url", CK_PORT, CK_IP, CK_PORT), new Properties());
 
         halfDatasource.actualize();
         Connection connection = halfDatasource.getConnection();
@@ -167,27 +168,29 @@ public void testConstructWithProperties() {
 
         // without connection parameters
         BalancedClickhouseDataSource dataSource = new BalancedClickhouseDataSource(
-                "jdbc:clickhouse://localhost:9000,127.0.0.1:9000/click", properties);
+                String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click", CK_HOST, CK_PORT, CK_IP, CK_PORT), properties);
         ClickHouseConfig cfg = dataSource.getCfg();
         assertEquals(Duration.ofSeconds(6789), cfg.queryTimeout());
         assertEquals("888888", cfg.password());
         assertEquals("click", cfg.database());
         assertEquals(2, dataSource.getAllClickhouseUrls().size());
-        assertEquals(dataSource.getAllClickhouseUrls().get(0), "jdbc:clickhouse://localhost:9000/click");
-        assertEquals(dataSource.getAllClickhouseUrls().get(1), "jdbc:clickhouse://127.0.0.1:9000/click");
+        assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click", CK_HOST, CK_PORT),
+                dataSource.getAllClickhouseUrls().get(0));
+        assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click", CK_IP, CK_PORT),
+                dataSource.getAllClickhouseUrls().get(1));
 
         // with connection parameters
         dataSource = new BalancedClickhouseDataSource(
-                "jdbc:clickhouse://localhost:9000,127.0.0.1:9000/click?query_timeout=12345&user=readonly", properties);
+                String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT, CK_IP, CK_PORT), properties);
         cfg = dataSource.getCfg();
         assertEquals(Duration.ofSeconds(6789), cfg.queryTimeout());
         assertEquals("readonly", cfg.user());
         assertEquals("888888", cfg.password());
         assertEquals("click", cfg.database());
         assertEquals(2, dataSource.getAllClickhouseUrls().size());
-        assertEquals("jdbc:clickhouse://localhost:9000/click?query_timeout=12345&user=readonly",
+        assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT),
                 dataSource.getAllClickhouseUrls().get(0));
-        assertEquals("jdbc:clickhouse://127.0.0.1:9000/click?query_timeout=12345&user=readonly",
+        assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click?query_timeout=12345&user=readonly", CK_IP, CK_PORT),
                 dataSource.getAllClickhouseUrls().get(1));
     }
 }
diff --git a/...ative-jdbc/src/test/java/com/github/housepower/jdbc/BalancedClickhouseDataSourceTest.java b/...ative-jdbc/src/test/java/com/github/housepower/jdbc/BalancedClickhouseDataSourceTest.java
@@ -46,7 +46,6 @@ public void testUrlSplitValidHostName() {
                 BalancedClickhouseDataSource.splitUrl("jdbc:clickhouse://localhost:1234,_0another-host.com:4321"));
     }
 
-
     @Test
     public void testUrlSplitInvalidHostName() {
         assertThrows(InvalidValueException.class, () ->

diff --git a/...native-jdbc/src/test/java/com/github/housepower/jdbc/ClickHouseDatabaseMetadataITest.java b/...native-jdbc/src/test/java/com/github/housepower/jdbc/ClickHouseDatabaseMetadataITest.java
@@ -21,6 +21,7 @@
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
 import java.sql.RowIdLifetime;
+import java.util.Locale;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -46,7 +47,8 @@ void allTablesAreSelectable() throws Exception {
     void getURL() throws Exception {
         withNewConnection(connection -> {
             DatabaseMetaData dm = connection.getMetaData();
-            assertEquals("jdbc:clickhouse://127.0.0.1:9000/default?query_timeout=0&connect_timeout=0&charset=UTF-8&tcp_keep_alive=false", dm.getURL());
+            assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/default?query_timeout=0&connect_timeout=0&charset=UTF-8&tcp_keep_alive=false", CK_HOST, CK_PORT),
+                    dm.getURL());
         });
     }
 

diff --git a/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/ConnectionParamITest.java b/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/ConnectionParamITest.java
@@ -21,6 +21,7 @@
 
 import java.sql.*;
 import java.time.Duration;
+import java.util.Locale;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -36,7 +37,7 @@ public void init() throws SQLException {
     public void successfullyMaxRowsToRead() {
         assertThrows(SQLException.class, () -> {
             Connection connection = DriverManager
-                    .getConnection("jdbc:clickhouse://127.0.0.1?max_rows_to_read=1&connect_timeout=10");
+                    .getConnection(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s?max_rows_to_read=1&connect_timeout=10", CK_HOST, CK_PORT));
             Statement statement = connection.createStatement();
             ResultSet rs = statement.executeQuery("SELECT arrayJoin([1,2,3,4]) from numbers(100)");
             int rowsRead = 0;
@@ -50,7 +51,7 @@ public void successfullyMaxRowsToRead() {
     @Test
     public void successfullyMaxResultRows() throws Exception {
         Connection connection = DriverManager
-                .getConnection("jdbc:clickhouse://127.0.0.1?max_result_rows=1&connect_timeout=10");
+                .getConnection(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s?max_result_rows=1&connect_timeout=10", CK_HOST, CK_PORT));
         Statement statement = connection.createStatement();
         statement.setMaxRows(400);
         ResultSet rs = statement.executeQuery("SELECT arrayJoin([1,2,3,4]) from numbers(100)");