Merge branch 'main' into wangzheng/refactor_notification

.github/workflows/assign-issue-milestone.yml

@@ -9,11 +9,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Add issue to the latest milestone
-      uses: cgetc/automatically-set-milestone-to-issue@v0.1.2
+      uses: fuyufjh/automatically-set-milestone-to-issue@main
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         version-prefix: 'release-'
         version-separator: '.'
+        overwrite: false
     - name: Add issue to project
       uses: actions/add-to-project@v0.3.0
       with:

Makefile.toml

@@ -647,6 +647,18 @@ set -e
 cargo run -p risingwave_simulation "$@"
 """
 
+[tasks.sslt-build-all]
+category = "RiseDev - Deterministic Simulation End-to-end Test"
+description = "Build deterministic simulation runner and tests"
+dependencies = ["warn-on-missing-tools"]
+env = { RUSTFLAGS = "-Ctarget-cpu=native --cfg tokio_unstable --cfg madsim", RUSTDOCFLAGS = "--cfg madsim", CARGO_TARGET_DIR = "target/sim" }
+script = """
+#!/bin/bash
+set -e
+
+cargo build -p risingwave_simulation --tests "$@"
+"""
+
 [tasks.sslt-cov]
 category = "RiseDev - Deterministic Simulation End-to-end Test"
 description = "Run e2e tests in deterministic simulation mode and report code coverage"

ci/scripts/build-other.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 source ci/scripts/common.env.sh
 
 # Should set a stable version of connector node
-STABLE_VERSION=9d2e1331680335661fd3e55ee234532358683201
+STABLE_VERSION=bd12fb55c75f09b234d1b75b8671b7582ca533f3
 
 echo "--- Build Java connector node"
 git clone https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git

ci/scripts/build-simulation.sh

@@ -9,12 +9,13 @@ echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Build deterministic simulation e2e test runner"
-cargo make sslt --profile ci-sim -- --help
+cargo make sslt-build-all --profile ci-sim
 
 echo "--- Build and archive deterministic scaling simulation tests"
 NEXTEST_PROFILE=ci-scaling cargo make sarchive-scale-test --cargo-profile ci-sim
 
 echo "--- Upload artifacts"
-cp target/sim/ci-sim/risingwave_simulation ./risingwave_simulation
-buildkite-agent artifact upload risingwave_simulation
-buildkite-agent artifact upload scale-test.tar.zst
+mv target/sim/ci-sim/risingwave_simulation ./risingwave_simulation
+
+artifacts=(risingwave_simulation scale-test.tar.zst)
+echo -n "${artifacts[*]}" | parallel -d ' ' "buildkite-agent artifact upload ./{}"

ci/scripts/build.sh

@@ -43,30 +43,13 @@ cargo build \
     -p risingwave_backup_cmd \
     --features "static-link static-log-level" --profile "$profile"
 
+artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risedev-dev delete-range-test)
+
 echo "--- Compress debug info for artifacts"
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/sqlsmith
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/compaction-test
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/backup-restore
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave_regress_test
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/risedev-dev
-objcopy --compress-debug-sections=zlib-gnu target/"$target"/delete-range-test
+echo -n "${artifacts[*]}" | parallel -d ' ' "objcopy --compress-debug-sections=zlib-gnu target/$target/{} && echo \"compressed {}\""
 
 echo "--- Show link info"
 ldd target/"$target"/risingwave
 
 echo "--- Upload artifacts"
-cp target/"$target"/compaction-test ./compaction-test-"$profile"
-cp target/"$target"/backup-restore ./backup-restore-"$profile"
-cp target/"$target"/risingwave ./risingwave-"$profile"
-cp target/"$target"/risedev-dev ./risedev-dev-"$profile"
-cp target/"$target"/risingwave_regress_test ./risingwave_regress_test-"$profile"
-cp target/"$target"/sqlsmith ./sqlsmith-"$profile"
-cp target/"$target"/delete-range-test ./delete-range-test-"$profile"
-buildkite-agent artifact upload risingwave-"$profile"
-buildkite-agent artifact upload risedev-dev-"$profile"
-buildkite-agent artifact upload risingwave_regress_test-"$profile"
-buildkite-agent artifact upload ./sqlsmith-"$profile"
-buildkite-agent artifact upload ./compaction-test-"$profile"
-buildkite-agent artifact upload ./backup-restore-"$profile"
-buildkite-agent artifact upload ./delete-range-test-"$profile"
+echo -n "${artifacts[*]}" | parallel -d ' ' "mv target/$target/{} ./{}-$profile && buildkite-agent artifact upload ./{}-$profile"

ci/scripts/check.sh

@@ -3,6 +3,18 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
+# Check ci bash scripts contains `set -euo pipefail`.
+for script in ci/**/*.sh; do
+    # skip .env.sh
+    if [[ "$script" == *".env.sh" ]]; then
+        continue
+    fi
+    if ! grep -Fq 'set -euo pipefail' "$script"; then
+        echo "ERROR: $script does not contain 'set -euo pipefail'"
+        exit 1
+    fi
+done
+
 source ci/scripts/common.env.sh
 
 echo "--- Run clippy check (dev, all features)"

ci/scripts/cron-e2e-test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Exits as soon as any line fails.
 set -euo pipefail

ci/scripts/cron-fuzz-test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Exits as soon as any line fails.
 set -euo pipefail

ci/scripts/deterministic-e2e-test.sh

@@ -9,6 +9,12 @@ echo "--- Download artifacts"
 buildkite-agent artifact download risingwave_simulation .
 chmod +x ./risingwave_simulation
 
+echo "--- Extract data for Kafka"
+cd ./scripts/source/
+mkdir -p ./test_data
+unzip -o test_data.zip -d ./test_data
+cd ../../
+
 export RUST_LOG=info
 export LOGDIR=.risingwave/log
 

ci/scripts/deterministic-recovery-test.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Exits as soon as any line fails.
+set -euo pipefail
+
 source ci/scripts/common.env.sh
 
 echo "--- Download artifacts"

ci/scripts/e2e-sink-test.sh

@@ -45,6 +45,10 @@ cargo make link-all-in-one-binaries
 # prepare environment mysql sink
 apt-get -y install mysql-client
 mysql --host=mysql --port=3306 -u root -p123456 -e "CREATE DATABASE IF NOT EXISTS test;"
+# grant access to `test` for ci test user
+mysql --host=mysql --port=3306 -u root -p123456 -e "GRANT ALL PRIVILEGES ON test.* TO 'mysqluser'@'%';"
+# create a table named t_remote
+mysql --host=mysql --port=3306 -u root -p123456 -e "CREATE TABLE IF NOT EXISTS test.t_remote (id INT, name VARCHAR(255), PRIMARY KEY (id));"
 
 echo "--- preparing postgresql"
 
@@ -64,11 +68,26 @@ sleep 1
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/*.slt'
 sleep 1
-sqllogictest -p 4566 -d dev './e2e_test/sink/remote/remote.load.slt'
+
+# check sink destination postgres
+sqllogictest -p 4566 -d dev './e2e_test/sink/remote/jdbc.load.slt'
+sleep 1
+sqllogictest -h db -p 5432 -d test './e2e_test/sink/remote/jdbc.check.pg.slt'
 sleep 1
 
-# check sink destination
-sqllogictest -h db -p 5432 -d test './e2e_test/sink/remote/remote.check.slt'
+# check sink destination mysql using shell
+if mysql  --host=mysql --port=3306 -u root -p123456 -sN -e "SELECT * FROM test.t_remote ORDER BY id;" | awk '{
+if ($1 == 1 && $2 == "Alex") c1++;
+ if ($1 == 3 && $2 == "Carl") c2++;
+  if ($1 == 4 && $2 == "Doris") c3++;
+   if ($1 == 5 && $2 == "Eve") c4++;
+    if ($1 == 6 && $2 == "Frank") c5++; }
+     END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1); }'; then
+  echo "mysql sink check passed"
+else
+  echo "The output is not as expected."
+  exit 1
+fi
 
 echo "--- Kill cluster"
 pkill -f connector-service.jar

ci/scripts/e2e-source-test.sh

@@ -66,6 +66,12 @@ nohup java -jar ./connector-service.jar --port 60061 > .risingwave/log/connector
 # start risingwave cluster
 cargo make ci-start ci-1cn-1fe-with-recovery
 sleep 2
+
+echo "---- mysql & postgres cdc validate test"
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.mysql.slt'
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.postgres.slt'
+
+echo "---- mysql & postgres load and check"
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt'
 # wait for cdc loading
 sleep 10
@@ -85,16 +91,22 @@ echo "check mviews after cluster recovery"
 # check results
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt'
 
-
 echo "--- Kill cluster"
 pkill -f connector-service.jar
 cargo make ci-kill
 
-echo "--- e2e test w/ Rust frontend - source with kafka and pubsub"
+echo "--- e2e, ci-1cn-1fe, nexmark endless"
+cargo make ci-start ci-1cn-1fe
+sqllogictest -p 4566 -d dev './e2e_test/source/nexmark_endless/*.slt'
+
+echo "--- Kill cluster"
+cargo make ci-kill
+
+echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source"
 cargo make ci-start ci-kafka-plus-pubsub
 ./scripts/source/prepare_ci_kafka.sh
 cargo run --bin prepare_ci_pubsub
-sqllogictest -p 4566 -d dev  './e2e_test/source/basic/*.slt'
+sqllogictest -p 4566 -d dev './e2e_test/source/basic/*.slt'
 
 echo "--- Run CH-benCHmark"
 ./risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt'

ci/scripts/pr.env.sh

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+# Exits as soon as any line fails.
 set -euo pipefail
 
 set +e

ci/scripts/run-e2e-test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Exits as soon as any line fails.
 set -euo pipefail

ci/scripts/run-fuzz-test.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+
+# Exits as soon as any line fails.
 set -euo pipefail
 
 if [[ "$RUN_SQLSMITH" -eq "1" ]]; then

ci/scripts/run-unit-test.sh

@@ -1,3 +1,7 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
 
 echo "+++ Run unit tests with coverage"
 # use tee to disable progress bar

dashboard/pages/data_sources.tsx

@@ -84,7 +84,7 @@ export default function DataSources() {
                   <Td>{source.id}</Td>
                   <Td>{source.name}</Td>
                   <Td>{source.owner}</Td>
-                  <Td>{source.info?.$case}</Td>
+                  <Td>{source.info}</Td>
                   <Td>
                     <Button
                       size="sm"