Merge branch 'main' into right_joi

.github/actions/test_stateful_cluster_linux/action.yml

@@ -0,0 +1,58 @@
+name: "Test Stateful Cluster Linux"
+description: "Running stateful tests in cluster mode"
+inputs:
+  profile:
+    description: "The profile for this test"
+    required: true
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        profile: ${{ inputs.profile }}
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d -p 9900:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/admin/data  --recursive
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.csv s3://testbucket/admin/data/ontime_200_v1.csv
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
+
+    - name: Run Stateful Tests with Cluster mode (ubuntu-latest only)
+      shell: bash
+      run: |
+        ./scripts/ci/ci-run-stateful-tests-cluster-s3.sh
+
+    - name: Upload .stdout files
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-stateful-cluster-linux-stdouts
+        path: |
+           ./tests/suites/1_stateful/*/*.stdout
+
+    - name: Upload failure
+      if: failure()
+      uses: ./.github/actions/artifact_failure
+      with:
+        name: test-stateful-cluster-linux

.github/mergify.yml

@@ -10,9 +10,9 @@ queue_rules:
       - check-success~=^build_(aarch64|x86_64)_musl$
       - check-success=test_unit
       - check-success=test_metactl
-      - check-success=test_stateless_standalone_linux
-      - check-success=test_stateless_cluster_linux
-      - check-success=test_stateful_standalone_linux
+      # state tests
+      - check-success~=^test_(stateless|stateful)_(standalone|cluster)_linux$
+      # sqllogic tests
       - check-success=test_sqllogic_base_standalone_linux
       - check-success=test_sqllogic_ydb_standalone_linux
       - check-success=test_sqllogic_base_cluster_linux

.github/workflows/dev-linux.yml

@@ -112,22 +112,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_meta_cluster
 
-  test_stateless_standalone_linux:
-    timeout-minutes: 10
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_standalone_linux
-
-  test_stateless_cluster_linux:
-    timeout-minutes: 15
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_cluster_linux
-
   test_sqllogic_standalone_linux:
     timeout-minutes: 30
     name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
@@ -161,14 +145,40 @@ jobs:
         with:
           dirs: ${{ matrix.dirs }}
 
-  test_stateful_standalone_linux:
-    timeout-minutes: 10
+  test_stateless_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_standalone_linux
+
+  test_stateless_cluster_linux:
+    timeout-minutes: 20
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_cluster_linux
+
+  test_stateful_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [ self-hosted, X64, Linux, development ]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_standalone_linux
 
+  # Wait for issue https://github.com/datafuselabs/databend/issues/7684
+  #
+  # test_stateful_cluster_linux:
+  #  timeout-minutes: 20
+  #  runs-on: [ self-hosted, X64, Linux, development ]
+  #  needs: build_gnu
+  #  steps:
+  #    - uses: actions/checkout@v3
+  #    - uses: ./.github/actions/test_stateful_cluster_linux
+
   test_fuzz_standalone_linux:
     timeout-minutes: 10
     runs-on: [self-hosted, X64, Linux, development]

scripts/ci/ci-run-stateful-tests-cluster-s3.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+echo "*************************************"
+echo "* Setting STORAGE_TYPE to S3.       *"
+echo "*                                   *"
+echo "* Please make sure that S3 backend  *"
+echo "* is ready, and configured properly.*"
+echo "*************************************"
+export STORAGE_TYPE=s3
+export STORAGE_S3_BUCKET=testbucket
+export STORAGE_S3_ROOT=admin
+export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+export STORAGE_ALLOW_INSECURE=true
+
+echo "Install dependence"
+python3 -m pip install --quiet mysql-connector-python
+
+echo "calling test suite"
+echo "Starting Cluster databend-query"
+./scripts/ci/deploy/databend-query-cluster-3-nodes.sh
+
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+cd "$SCRIPT_PATH/../../tests" || exit
+
+echo "Starting databend-test"
+./databend-test --mode 'cluster' --run-dir 1_stateful