Merge branch 'main' into refactor/functional-scalar-function-binder

risingwavelabs · Jan 28, 2023 · 56adea4 · 56adea4
2 parents 57d7923 + 6c34da7
commit 56adea4
Show file tree

Hide file tree

Showing 98 changed files with 1,953 additions and 1,231 deletions.
diff --git a/.mergify.yml b/.mergify.yml
@@ -22,23 +22,6 @@ queue_rules:
       - "label=mergify/can-merge"
 
 pull_request_rules:
-  - name: Automatic merge on approval (high priority)
-    conditions:
-      - "base=main"
-      - "#approved-reviews-by>=1"
-      - "#changes-requested-reviews-by=0"
-      - "check-success=buildkite/pull-request"
-      - "check-failure!=buildkite/docker"
-      - "check-success=task-list-completed"
-      - "label=mergify/can-merge"
-      - "label=mergify/high-priority"
-    actions:
-      queue:
-        name: default
-        method: squash
-        priority: high
-        commit_message_template: *commit_message_template
-
   - name: Automatic merge on approval
     conditions:
       - "base=main"
@@ -52,7 +35,6 @@ pull_request_rules:
       queue:
         name: default
         method: squash
-        priority: medium
         commit_message_template: *commit_message_template
 
   - name: Notify author on queue failure

diff --git a/Makefile.toml b/Makefile.toml
@@ -501,16 +501,24 @@ script = """
 
 set -e
 
-for tool in cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make cargo-upgrades typos-cli
+cargo install cargo-quickinstall --locked
+cargo quickinstall cargo-binstall
+
+for tool in cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make
 do
   echo "install: $(tput setaf 4)$tool$(tput sgr0)"
-  cargo install $tool --locked
+  cargo binstall -y --no-symlinks $tool
   echo
 done
 
-echo "install: $(tput setaf 4)risinglightdb/sqllogictest-rs$(tput sgr0)"
-cargo install --git https://github.com/risinglightdb/sqllogictest-rs --bin sqllogictest
-echo
+# Tools that fallback to `cargo install` when using `cargo binstall`.
+# We directly use `cargo install` here to be faster.
+for tool in typos-cli sqllogictest-bin
+do
+  echo "install: $(tput setaf 4)$tool$(tput sgr0)"
+  cargo install $tool --locked
+  echo
+done
 
 echo "check: $(tput setaf 4)tmux >= v3.2a$(tput sgr0)"
 tmux -V || echo "$(tput setaf 3)tmux$(tput sgr0) not found."
@@ -528,7 +536,7 @@ echo "check: $(tput setaf 4)protoc$(tput sgr0)"
 protoc --version || echo "$(tput setaf 3)protoc$(tput sgr0) not found."
 echo
 """
-description = "Install required tools to do pre-CI check and run e2e tests"
+description = "Install (or upgrade) required tools to do pre-CI check and run e2e tests"
 
 [tasks.warn-on-missing-tools]
 private = true

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -6,7 +6,7 @@ ARG RUST_TOOLCHAIN
 
 RUN apt-get update -yy && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel \
-    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat \
+    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \
     maven -yy \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
 

diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
@@ -14,7 +14,7 @@ export RUST_TOOLCHAIN=$(cat ../rust-toolchain)
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20230119-1
+export BUILD_ENV_VERSION=v20230126
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -30,28 +30,28 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230119-1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230126
     depends_on:
       - mysql
       - db
     volumes:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230119-1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230126
     depends_on:
       - mysql
       - db
     volumes:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230119-1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230126
     volumes:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230119-1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230126
     depends_on:
       db:
         condition: service_healthy

diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh
@@ -20,16 +20,16 @@ export LOGDIR=.risingwave/log
 
 mkdir -p $LOGDIR
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, ddl"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, ddl"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation ./e2e_test/ddl/\*\*/\*.slt > $LOGDIR/ddl-{}.log && rm $LOGDIR/ddl-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, streaming"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, streaming"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation ./e2e_test/streaming/\*\*/\*.slt > $LOGDIR/streaming-{}.log && rm $LOGDIR/streaming-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, batch"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, kafka source"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, kafka source"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kafka-datadir=./scripts/source/test_data ./e2e_test/source/kafka.slt > $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming"
@@ -38,5 +38,5 @@ seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, fuzzing"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata > $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh
@@ -14,8 +14,8 @@ export LOGDIR=.risingwave/log
 
 mkdir -p $LOGDIR
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, streaming"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, recovery, streaming"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/streaming/\*\*/\*.slt > $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
 
-echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, batch"
+echo "--- deterministic simulation e2e, ci-3cn-2fe, recovery, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
diff --git a/ci/scripts/docker.sh b/ci/scripts/docker.sh
@@ -14,10 +14,12 @@ echo "--- docker build and tag"
 docker build -f docker/Dockerfile -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" --target risingwave .
 
 echo "--- check the image can start correctly"
-container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" risingwave)
-ret_code=$(docker inspect --format='{{.State.ExitCode}}' "$container_id")
-if [ "$ret_code" -ne 0 ]; then
-  echo "docker run failed with exit code $ret_code"
+container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" playground)
+sleep 10
+container_status=$(docker inspect --format='{{.State.Status}}' "$container_id")
+if [ "$container_status" != "running" ]; then
+  echo "docker run failed with status $container_status"
+  docker inspect "$container_id"
   docker logs "$container_id"
   exit 1
 fi
@@ -36,4 +38,4 @@ docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
 
 echo "--- docker push to dockerhub"
 docker tag "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
-docker push "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
+docker push "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh
@@ -43,7 +43,6 @@ cargo make pre-start-dev
 cargo make link-all-in-one-binaries
 
 # prepare environment mysql sink
-apt-get -y install mysql-client
 mysql --host=mysql --port=3306 -u root -p123456 -e "CREATE DATABASE IF NOT EXISTS test;"
 # grant access to `test` for ci test user
 mysql --host=mysql --port=3306 -u root -p123456 -e "GRANT ALL PRIVILEGES ON test.* TO 'mysqluser'@'%';"

diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
@@ -51,8 +51,7 @@ cargo make pre-start-dev
 cargo make link-all-in-one-binaries
 
 echo "--- e2e, ci-1cn-1fe, mysql & postgres cdc"
-# install mysql client
-apt-get -y install mysql-client
+
 # import data to mysql
 mysql --host=mysql --port=3306 -u root -p123456 < ./e2e_test/source/cdc/mysql_cdc.sql
 

diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json
diff --git a/dashboard/proto/gen/common.ts b/dashboard/proto/gen/common.ts
diff --git a/dashboard/proto/gen/meta.ts b/dashboard/proto/gen/meta.ts