Merge branch 'main' into chore/keep-retry-recovery

ci/Dockerfile

@@ -6,7 +6,8 @@ ARG RUST_TOOLCHAIN
 
 RUN apt-get update -yy && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel \
-    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat -yy \
+    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat \
+    maven -yy \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 SHELL ["/bin/bash", "-c"]

ci/build-ci-image.sh

@@ -14,7 +14,7 @@ export RUST_TOOLCHAIN=$(cat ../rust-toolchain)
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20221105
+export BUILD_ENV_VERSION=v20221122
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -14,6 +14,20 @@ services:
       timeout: 5s
       retries: 5
 
+  mysql:
+    image: mysql:8.0
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=123456
+      - MYSQL_USER=mysqluser
+      - MYSQL_PASSWORD=mysqlpw
+    healthcheck:
+      test: [ "CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p123456" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   zookeeper:
     image: confluentinc/cp-zookeeper
     ports:
@@ -41,21 +55,28 @@ services:
       timeout: 10s
       retries: 5
 
+  source-test-env:
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221122
+    depends_on:
+      - mysql
+    volumes:
+      - ..:/risingwave
+
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221105
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221122
     volumes:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221105
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221122
     depends_on:
       db:
         condition: service_healthy
     volumes:
       - ..:/risingwave
 
   benchmark-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221105
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20221122
     depends_on:
       - kafka
     volumes:

ci/scripts/build-other.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+echo "--- Build Java connector node"
+# clone a released version(tag)
+git clone --branch v0.0.1 --depth 1 https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git
+cd risingwave-connector-node
+mvn package -Dmaven.test.skip=true
+echo "--- Upload Java artifacts"
+cp service/target/service-*.jar ./connector-service.jar
+buildkite-agent artifact upload ./connector-service.jar

ci/scripts/build.sh

@@ -42,8 +42,12 @@ cargo build \
     -p risingwave_compaction_test \
     --features "static-link static-log-level" --profile "$profile"
 
-echo "--- Compress RisingWave debug info"
+echo "--- Compress debug info for artifacts"
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave
+objcopy --compress-debug-sections=zlib-gnu target/"$target"/sqlsmith
+objcopy --compress-debug-sections=zlib-gnu target/"$target"/compaction-test
+objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave_regress_test
+objcopy --compress-debug-sections=zlib-gnu target/"$target"/risedev-dev
 
 echo "--- Show link info"
 ldd target/"$target"/risingwave

ci/scripts/e2e-source-test.sh

@@ -28,6 +28,9 @@ buildkite-agent artifact download risedev-dev-"$profile" target/debug/
 mv target/debug/risingwave-"$profile" target/debug/risingwave
 mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
 
+echo "--- Download connector node jar"
+buildkite-agent artifact download connector-service.jar ./
+
 echo "--- Prepare data"
 cp src/source/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
 cp src/source/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
@@ -44,11 +47,31 @@ echo "--- Prepare RiseDev dev cluster"
 cargo make pre-start-dev
 cargo make link-all-in-one-binaries
 
+echo "--- e2e, ci-1cn-1fe, cdc source"
+# install mysql client
+apt-get -y install mysql-client
+# import data to mysql
+mysql --host=mysql --port=3306 -u root -p123456 < ./e2e_test/source/cdc/mysql_cdc.sql
+# start risingwave cluster
+cargo make ci-start ci-1cn-1fe
+# start cdc connector node
+nohup java -cp ./connector-service.jar com.risingwave.sourcenode.service.SourceServiceMain > .risingwave/log/connector-source.log 2>&1 &
+sleep 1
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt'
+# wait for cdc loading
+sleep 4
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
+
+echo "--- Kill cluster"
+pkill -f connector-service.jar
+cargo make ci-kill
+
 echo "--- e2e test w/ Rust frontend - source with kafka"
 cargo make clean-data
 cargo make ci-start ci-kafka
 ./scripts/source/prepare_ci_kafka.sh
-sqllogictest -p 4566 -d dev  './e2e_test/source/**/*.slt'
+sqllogictest -p 4566 -d dev  './e2e_test/source/basic/*.slt'
+
 
 echo "--- Run CH-benCHmark"
 ./risedev slt -p 4566 -d dev ./e2e_test/ch-benchmark/ch_benchmark.slt

ci/workflows/main-cron.yml

@@ -32,6 +32,22 @@ steps:
     timeout_in_minutes: 20
     retry: *auto-retry
 
+  - label: "build other components"
+    command: "ci/scripts/build-other.sh"
+    key: "build-other"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - GITHUB_TOKEN
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "build (deterministic simulation)"
     command: "ci/scripts/build-simulation.sh"
     key: "build-simulation"
@@ -73,11 +89,13 @@ steps:
 
   - label: "end-to-end source test (release)"
     command: "ci/scripts/e2e-source-test.sh -p ci-release"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "build-other"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs

ci/workflows/main.yml

@@ -44,6 +44,22 @@ steps:
     timeout_in_minutes: 20
     retry: *auto-retry
 
+  - label: "build other components"
+    command: "ci/scripts/build-other.sh"
+    key: "build-other"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - GITHUB_TOKEN
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
     key: "docslt"
@@ -157,11 +173,13 @@ steps:
 
   - label: "end-to-end source test (release mode)"
     command: "ci/scripts/e2e-source-test.sh -p ci-release"
-    depends_on: "build-release"
+    depends_on:
+      - "build-release"
+      - "build-other"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs

ci/workflows/pull-request.yml

@@ -43,6 +43,22 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "build other components"
+    command: "ci/scripts/build-other.sh"
+    key: "build-other"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v3.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - GITHUB_TOKEN
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "build (deterministic simulation)"
     command: "ci/scripts/build-simulation.sh"
     key: "build-simulation"
@@ -112,11 +128,13 @@ steps:
 
   - label: "end-to-end source test"
     command: "ci/scripts/e2e-source-test.sh -p ci-dev"
-    depends_on: "build"
+    depends_on:
+      - "build"
+      - "build-other"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs

e2e_test/source/nexmark_endless.slt → e2e_test/source/basic/nexmark_endless.slt

@@ -1,9 +1,9 @@
-include ../nexmark/create_sources.slt.part
+include ../../nexmark/create_sources.slt.part
 
 # Note: It will take a very long time to create all of the NexMark materialized views,
 # so we just create Q5 and Q7 here
-include ../streaming/nexmark/views/q7.slt.part
-include ../streaming/nexmark/views/q5.slt.part
+include ../../streaming/nexmark/views/q7.slt.part
+include ../../streaming/nexmark/views/q5.slt.part
 
 sleep 15s
 

e2e_test/source/cdc/cdc.check.slt

@@ -0,0 +1,23 @@
+# CDC source basic test
+
+query I
+select cnt from products_cnt;
+----
+9
+
+query II
+select cnt from orders_cnt;
+----
+3
+
+query III
+select cnt from shipments_cnt;
+----
+3
+
+query IIII
+select order_id, product_id, shipment_id from enriched_orders order by order_id;
+----
+10001  102   1001
+10002  105   1002
+10003  106   1003