Merge branch 'apache:develop' into fix_heart_rq_mode

apache · Oct 19, 2024 · bd5ff58 · bd5ff58
2 parents 93d9bf7 + 1e0b08f
commit bd5ff58
Show file tree

Hide file tree

Showing 129 changed files with 3,035 additions and 743 deletions.
diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml
@@ -8,9 +8,6 @@ on:
  - develop
  - bazel
 
-permissions:
- actions: write
-
 jobs:
  build:
  name: "bazel-compile (${{ matrix.os }})"
@@ -23,14 +20,4 @@ jobs:
  - name: Build
  run: bazel build --config=remote //...
  - name: Run Tests
- run: bazel test --config=remote //...
- - name: Retry if failed
- # if it failed , retry 2 times at most
- if: failure() && fromJSON(github.run_attempt) < 3
- continue-on-error: true
- env:
- GH_REPO: ${{ github.repository }}
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- echo "Attempting to retry workflow..."
- gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }}
+ run: bazel test --config=remote //...
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -35,4 +35,11 @@ jobs:
  - name: Run integration tests with Maven
  run: mvn clean verify -Pit-test -Pskip-unit-tests
 
-
+ - name: Publish Test Report
+ uses: mikepenz/action-junit-report@v3
+ if: always()
+ with:
+ report_paths: 'test/target/failsafe-reports/TEST-*.xml'
+ annotate_only: true
+ include_passed: true
+ detailed_summary: true
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
@@ -5,9 +5,6 @@ on:
  push:
  branches: [master, develop, bazel]
 
-permissions:
- actions: write
-
 jobs:
  java_build:
  name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})"
@@ -44,15 +41,4 @@ jobs:
  with:
  name: jvm-crash-logs
  path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log
- retention-days: 1
-
- - name: Retry if failed
- # if it failed , retry 2 times at most
- if: failure() && fromJSON(github.run_attempt) < 3
- continue-on-error: true
- env:
- GH_REPO: ${{ github.repository }}
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- echo "Attempting to retry workflow..."
- gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }}
+ retention-days: 1
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
@@ -21,7 +21,7 @@ jobs:
  - name: Build distribution tar
  run: |
  mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  name: Upload distribution tar
  with:
  name: rocketmq
@@ -30,7 +30,7 @@ jobs:
  run: |
  mkdir -p ./pr
  echo ${{ github.event.number }} > ./pr/NR
- - uses: actions/upload-artifact@v2
+ - uses: actions/upload-artifact@v4
  with:
  name: pr
  path: pr/
diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml
@@ -25,18 +25,18 @@ jobs:
  java-version: ["8"]
  steps:
  - name: 'Download artifact'
- uses: actions/github-script@v3.1.0
+ uses: actions/github-script@v6
  with:
  script: |
- var artifacts = await github.actions.listWorkflowRunArtifacts({
+ let artifacts = await github.rest.actions.listWorkflowRunArtifacts({
  owner: context.repo.owner,
  repo: context.repo.repo,
  run_id: ${{github.event.workflow_run.id }},
  });
- var matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => {
+ let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => {
  return artifact.name == "rocketmq"
  })[0];
- var download = await github.actions.downloadArtifact({
+ let download = await github.rest.actions.downloadArtifact({
  owner: context.repo.owner,
  repo: context.repo.repo,
  artifact_id: matchArtifactRmq.id,
@@ -68,7 +68,7 @@ jobs:
  mkdir versionlist
  touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
  sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  name: Upload distribution tar
  with:
  name: versionlist
@@ -85,7 +85,7 @@ jobs:
  outputs:
  version-json: ${{ steps.show_versions.outputs.version-json }}
  steps:
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
  name: Download versionlist
  with:
  name: versionlist
@@ -96,6 +96,7 @@ jobs:
  a=(`ls versionlist`)
  printf '%s\n' "${a[@]}" | jq -R . | jq -s .
  echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
+
  deploy:
  if: ${{ success() }}
  name: Deploy RocketMQ
@@ -158,7 +159,7 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
@@ -199,7 +200,7 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
@@ -235,7 +236,7 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
@@ -258,5 +259,4 @@ jobs:
  action: "clean"
  ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
  test-version: "${{ matrix.version }}"
- job-id: ${{ strategy.job-index }}
-
+ job-id: ${{ strategy.job-index }}
diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml
@@ -31,7 +31,7 @@ jobs:
  - name: Build distribution tar
  run: |
  mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  name: Upload distribution tar
  with:
  name: rocketmq
@@ -53,7 +53,7 @@ jobs:
  repository: apache/rocketmq-docker.git
  ref: master
  path: rocketmq-docker
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
  name: Download distribution tar
  with:
  name: rocketmq
@@ -72,13 +72,12 @@ jobs:
  mkdir versionlist
  touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
  sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  name: Upload distribution tar
  with:
  name: versionlist
  path: rocketmq-docker/image-build-ci/versionlist/*
 
-
  list-version:
  if: >
  github.repository == 'apache/rocketmq' && 
@@ -90,7 +89,7 @@ jobs:
  outputs:
  version-json: ${{ steps.show_versions.outputs.version-json }}
  steps:
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
  name: Download versionlist
  with:
  name: versionlist
@@ -101,9 +100,10 @@ jobs:
  a=(`ls versionlist`)
  printf '%s\n' "${a[@]}" | jq -R . | jq -s .
  echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
- deploy:
+
+ deploy-e2e:
  if: ${{ success() }}
- name: Deploy RocketMQ
+ name: Deploy RocketMQ For E2E
  needs: [list-version,docker]
  runs-on: ubuntu-latest
  timeout-minutes: 60
@@ -134,10 +134,45 @@ jobs:
  image:
  repository: ${{env.DOCKER_REPO}}
  tag: ${{ matrix.version }}
+
+ deploy-benchmark:
+ if: ${{ success() }}
+ name: Deploy RocketMQ For Benchmarking
+ needs: [list-version,docker]
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+ steps:
+ - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
+ name: Deploy rocketmq
+ with:
+ action: "deploy"
+ ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+ test-version: "${{ matrix.version }}"
+ chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git"
+ chart-branch: "master"
+ chart-path: "./rocketmq-k8s-helm"
+ job-id: "001-${{ strategy.job-index }}"
+ helm-values: |
+ nameserver:
+ image:
+ repository: ${{env.DOCKER_REPO}}
+ tag: ${{ matrix.version }}
+ broker:
+ image:
+ repository: ${{env.DOCKER_REPO}}
+ tag: ${{ matrix.version }}
+ proxy:
+ image:
+ repository: ${{env.DOCKER_REPO}}
+ tag: ${{ matrix.version }}
+
  test-e2e-grpc-java:
  if: ${{ success() }}
  name: Test E2E grpc java
- needs: [list-version, deploy]
+ needs: [list-version, deploy-e2e]
  runs-on: ubuntu-latest
  timeout-minutes: 60
  strategy:
@@ -163,7 +198,7 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
@@ -173,7 +208,7 @@ jobs:
  test-e2e-golang:
  if: ${{ success() }}
  name: Test E2E golang
- needs: [list-version, deploy]
+ needs: [list-version, deploy-e2e]
  runs-on: ubuntu-latest
  timeout-minutes: 60
  strategy:
@@ -204,7 +239,7 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
@@ -214,7 +249,7 @@ jobs:
  test-e2e-remoting-java:
  if: ${{ success() }}
  name: Test E2E remoting java
- needs: [ list-version, deploy ]
+ needs: [ list-version, deploy-e2e ]
  runs-on: ubuntu-latest
  timeout-minutes: 60
  strategy:
@@ -240,17 +275,45 @@ jobs:
  annotate_only: true
  include_passed: true
  detailed_summary: true
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  name: Upload test log
  with:
  name: test-e2e-remoting-java-log.txt
  path: testlog.txt
 
- clean:
+ benchmark-test:
+ if: ${{ success() }}
+ runs-on: ubuntu-latest
+ name: Performance benchmark test
+ needs: [ list-version, deploy-benchmark ]
+ timeout-minutes: 60
+ steps:
+ - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e
+ name: Performance benchmark
+ with:
+ action: "performance-benchmark"
+ ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+ job-id: "001-${{ strategy.job-index }}"
+ # The time to run the test, 15 minutes
+ test-time: "900"
+ # Some thresholds set in advance
+ min-send-tps-threshold: "12000"
+ max-rt-ms-threshold: "500"
+ avg-rt-ms-threshold: "10"
+ max-2c-rt-ms-threshold: "150"
+ avg-2c-rt-ms-threshold: "10"
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: benchmark-report
+ path: benchmark/
+
+ clean-e2e:
  if: always()
- name: Clean
- needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java]
+ name: Clean E2E
+ needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ]
  runs-on: ubuntu-latest
  timeout-minutes: 60
  strategy:
@@ -265,3 +328,20 @@ jobs:
  test-version: "${{ matrix.version }}"
  job-id: ${{ strategy.job-index }}
 
+ clean-benchmark:
+ if: always()
+ name: Clean Benchmarking
+ needs: [ list-version, benchmark-test ]
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+ steps:
+ - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
+ name: clean
+ with:
+ action: "clean"
+ ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+ test-version: "${{ matrix.version }}"
+ job-id: "001-${{ strategy.job-index }}"