diff --git a/.ci/build.groovy b/.ci/build.groovy deleted file mode 100644 index e8ddbdd00eb..00000000000 --- a/.ci/build.groovy +++ /dev/null @@ -1,63 +0,0 @@ -def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) { - - catchError { - node ('build') { - container("java") { - stage('Prepare') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi - """ - if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - deleteDir() - } - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]] - sh "git checkout -f ${ghprbActualCommit}" - } - } - - stage('Format') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - ./dev/javafmt - git diff --quiet - formatted="\$?" - if [[ "\${formatted}" -eq 1 ]] - then - echo "code format error, please run the following commands:" - echo " mvn com.coveo:fmt-maven-plugin:format" - exit 1 - fi - """ - } - } - - stage('Build') { - dir("/home/jenkins/agent/git/client-java") { - timeout(30) { - sh ".ci/build.sh" - } - } - } - } - } - currentBuild.result = "SUCCESS" - } - - stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Build Result: `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - print msg - } -} - -return this diff --git a/.ci/build.sh b/.ci/build.sh deleted file mode 100755 index 9be12fe6a53..00000000000 --- a/.ci/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -x -set -euo pipefail - -mvn clean compile \ No newline at end of file diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy deleted file mode 100644 index fac1bd92468..00000000000 --- a/.ci/integration_test.groovy +++ /dev/null @@ -1,117 +0,0 @@ -def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) { - - def TIKV_BRANCH = "master" - def PD_BRANCH = "master" - - // parse pd branch - def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m2) { - PD_BRANCH = "${m2[0][1]}" - } - m2 = null - println "PD_BRANCH=${PD_BRANCH}" - - // parse tikv branch - def m3 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m3) { - TIKV_BRANCH = "${m3[0][1]}" - } - m3 = null - println "TIKV_BRANCH=${TIKV_BRANCH}" - - catchError { - node ('build') { - container("java") { - stage('Prepare') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi - """ - if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - deleteDir() - } - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]] - sh "git checkout -f ${ghprbActualCommit}" - } - - dir("/home/jenkins/agent/git/client-java/_run") { - // tikv - def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim() - sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz" - // pd - def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim() - sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz" - - sh """ - killall -9 tikv-server || true - killall -9 pd-server || true - killall -9 java || true - sleep 10 - """ - - sh """ - echo "start TiKV for RawKV test" - bin/pd-server --name=pd_rawkv --data-dir=pd_rawkv --client-urls="http://0.0.0.0:2379" --advertise-client-urls="http://127.0.0.1:2379" --peer-urls="http://0.0.0.0:2380" --advertise-peer-urls="http://127.0.0.1:2380" --config=../config/pd.toml &>pd_rawkv.log & - sleep 10 - bin/tikv-server --pd 127.0.0.1:2379 --data-dir tikv_rawkv --addr 0.0.0.0:20160 --advertise-addr 127.0.0.1:20160 --status-addr 0.0.0.0:20180 --config ../config/tikv_rawkv.toml &>tikv_rawkv.log & - sleep 10 - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - """ - - sh """ - echo "start TiKV for TxnKV test" - bin/pd-server --name=pd_txnkv --data-dir=pd_txnkv --client-urls="http://0.0.0.0:3379" --advertise-client-urls="http://127.0.0.1:3379" --peer-urls="http://0.0.0.0:3380" --advertise-peer-urls="http://127.0.0.1:3380" --config=../config/pd.toml &>pd_txnkv.log & - sleep 10 - bin/tikv-server --pd 127.0.0.1:3379 --data-dir tikv_txnkv --addr 0.0.0.0:21160 --advertise-addr 127.0.0.1:21160 --status-addr 0.0.0.0:21180 --config ../config/tikv_txnkv.toml &>tikv_txnkv.log & - sleep 10 - ps aux | grep '-server' || true - curl -s 127.0.0.1:3379/pd/api/v1/status || true - """ - - sh "sleep 30" - } - } - - stage('Test') { - dir("/home/jenkins/agent/git/client-java") { - try { - timeout(30) { - sh ".ci/test.sh" - } - } catch (err) { - sh """ - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - """ - sh "cat _run/pd_rawkv.log" - sh "cat _run/tikv_rawkv.log" - sh "cat _run/pd_txnkv.log" - sh "cat _run/tikv_txnkv.log" - throw err - } - } - } - } - } - currentBuild.result = "SUCCESS" - } - - stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Integration Common Test Result: `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - - print msg - } -} - -return this diff --git a/.ci/test.sh b/.ci/test.sh deleted file mode 100755 index 6ba95963458..00000000000 --- a/.ci/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -x -set -euo pipefail - -export RAWKV_PD_ADDRESSES=127.0.0.1:2379 -export TXNKV_PD_ADDRESSES=127.0.0.1:3379 -mvn clean test \ No newline at end of file diff --git a/config/pd.toml b/.github/config/pd.toml similarity index 79% rename from config/pd.toml rename to .github/config/pd.toml index f2795de343f..526fa3fa745 100644 --- a/config/pd.toml +++ b/.github/config/pd.toml @@ -1,4 +1,4 @@ # PD Configuration. [replication] enable-placement-rules = true -max-replicas = 1 \ No newline at end of file +max-replicas = 1 diff --git a/.github/config/tikv_rawkv.toml b/.github/config/tikv_rawkv.toml new file mode 100644 index 00000000000..5ee1bfe07ed --- /dev/null +++ b/.github/config/tikv_rawkv.toml @@ -0,0 +1,18 @@ +# TiKV Configuration. + +[raftstore] +# set store capacity, if no set, use disk capacity. +capacity = "8G" +pd-heartbeat-tick-interval = "2s" +pd-store-heartbeat-tick-interval = "5s" +split-region-check-tick-interval = "1s" + +[storage] +enable-ttl = true + +[rocksdb] +max-open-files = 10000 + +[raftdb] +max-open-files = 10000 + diff --git a/.github/config/tikv_txnkv.toml b/.github/config/tikv_txnkv.toml new file mode 100644 index 00000000000..c083cfa31b0 --- /dev/null +++ b/.github/config/tikv_txnkv.toml @@ -0,0 +1,14 @@ +# TiKV Configuration. + +[raftstore] +# set store capacity, if no set, use disk capacity. +capacity = "8G" +pd-heartbeat-tick-interval = "2s" +pd-store-heartbeat-tick-interval = "5s" +split-region-check-tick-interval = "1s" + +[rocksdb] +max-open-files = 10000 + +[raftdb] +max-open-files = 10000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..c2fbf68aa88 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,71 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + +jobs: + format: + name: Check Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8.0' + distribution: 'adopt' + - run: | + ./dev/javafmt + if [[ $(git diff) != "" ]] + then + echo "code format error, please run the following commands:" + echo " ./dev/javafmt" + exit 1 + fi + + integration-test: + name: Integration Test - ${{ matrix.tikv_version }} + runs-on: ubuntu-latest + strategy: + matrix: + tikv_version: [nightly, v5.0.4, v5.3.0, v5.4.0] + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8.0' + distribution: 'adopt' + - name: Install TiUP + run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + - name: Start TiUP Playground + run: | + # Start TiKV in APIV1TTL + /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & + + # The first run of `tiup` has to download all components so it'll take longer. + sleep 1m 30s + + # Start TiKV in APIV1 + /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & + + sleep 30s + + # Parse PD address from `tiup` output + echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + + # Log the output + echo "$(cat raw.out)" >&2 + echo "$(cat txn.out)" >&2 + - name: Run Integration Test + run: mvn clean test + - name: Upload coverage + uses: codecov/codecov-action@v2 + with: + files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml + fail_ci_if_error: true + verbose: true diff --git a/README.md b/README.md index 065a5869bd1..0acd816deb1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Maven Central](https://img.shields.io/maven-central/v/org.tikv/tikv-client-java.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.tikv%22%20AND%20a:%22tikv-client-java%22) [![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://slack.tidb.io/invite?team=tikv-wg&channel=client) +[![codecov](https://codecov.io/gh/tikv/client-java/branch/master/graph/badge.svg?token=nSAjGaN0EH)](https://codecov.io/gh/tikv/client-java) ## TiKV JAVA Client diff --git a/config/tikv_rawkv.toml b/config/tikv_rawkv.toml deleted file mode 100644 index f525fa753c5..00000000000 --- a/config/tikv_rawkv.toml +++ /dev/null @@ -1,8 +0,0 @@ -# TiKV Configuration. - -[raftstore] -# set store capacity, if no set, use disk capacity. -capacity = "8G" - -[storage] -enable-ttl = true diff --git a/config/tikv_txnkv.toml b/config/tikv_txnkv.toml deleted file mode 100644 index 287abc69e41..00000000000 --- a/config/tikv_txnkv.toml +++ /dev/null @@ -1,5 +0,0 @@ -# TiKV Configuration. - -[raftstore] -# set store capacity, if no set, use disk capacity. -capacity = "8G" diff --git a/pom.xml b/pom.xml index b505ff9076e..23d6e934ce8 100644 --- a/pom.xml +++ b/pom.xml @@ -495,7 +495,7 @@ jacoco-site - package + test report