CI: backport ci to release-1.1 (#4963)

.github/workflows/integrationtests.yml

@@ -5,18 +5,22 @@ on:
  branches:
  - 'main'
  - 'release-*'
- paths-ignore:
- - 'docs/**'
- - '**.md'
+ paths:
+ - '**.c'
+ - '**.go'
+ - 'Makefile'
+ - '**/integrationtests.yml'
  pull_request:
- #The branches below must be a subset of the branches above
  branches:
  - 'main'
  - 'release-*'
- paths-ignore:
- - 'docs/**'
- - '**.md'
- - '.github/**'
+ paths:
+ - '**.c'
+ - '**.go'
+ - 'Makefile'
+ - '**/integrationtests.yml'
+ schedule:
+ - cron: '0 19 * * *'
  workflow_dispatch:
  inputs:
  debug:
@@ -26,89 +30,159 @@ on:
  default: false
 
 jobs:
+ build-matrix:
+ runs-on: ubuntu-20.04
+ steps:
+ - id: set-matrix
+ run: |
+ echo "github.event_name is ${{github.event_name}}"
+ echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
+ if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then
+ echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+ elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then
+ echo 'meta_matrix=["redis", "mysql", "tikv"]' >> $GITHUB_OUTPUT
+ else
+ echo "event_name is not supported" && exit 1
+ fi
+ outputs:
+ meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
+
  integrationtests:
- timeout-minutes: 60
+ timeout-minutes: 120
  runs-on: ubuntu-20.04
+ needs: build-matrix
+ strategy:
+ fail-fast: false
+ matrix:
+ meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
  steps:
  - name: Checkout
  uses: actions/checkout@v3
  with:
  fetch-depth: 1
 
+ - name: Remove unused software
+ timeout-minutes: 10
+ run: |
+ echo "before remove unused software"
+ sudo df -h
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /usr/local/lib/android
+ sudo rm -rf /opt/ghc
+ echo "after remove unused software"
+ sudo df -h
+ 
+ - name: Set Variable
+ id: vars
+ run: |
+ if [ "${{matrix.meta}}" == "fdb" ]; then
+ echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
+ else
+ echo "target=juicefs" >> $GITHUB_OUTPUT
+ fi
+
  - name: Build
  uses: ./.github/actions/build
+ with: 
+ target: ${{steps.vars.outputs.target}}
 
- - name: Copy
- run: |
- cp juicefs /tmp/mount.juicefs
-
- - name: Run Redis
- run: |
- sudo docker run -d --name redis -v redis-data:/data \
- -p 6379:6379 redis redis-server --appendonly yes
+ - name: Prepare meta db
+ run: | 
+ chmod +x .github/scripts/start_meta_engine.sh
+ source .github/scripts/start_meta_engine.sh
+ start_meta_engine ${{matrix.meta}}
+ meta_url=$(get_meta_url ${{matrix.meta}})
+ create_database $meta_url
 
  - name: Juicefs Format
  run: |
- sudo ./juicefs format redis://127.0.0.1:6379/1 pics
+ source .github/scripts/start_meta_engine.sh
+ meta_url=$(get_meta_url ${{matrix.meta}})
+ sudo ./juicefs format $meta_url --trash-days 0 pics
 
  - name: Juicefs Mount
  run: |
- sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report &
+ source .github/scripts/start_meta_engine.sh
+ meta_url=$(get_meta_url ${{matrix.meta}})
+ sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr
+ stat /jfs/.accesslog
 
  - name: Fslock Test
+ timeout-minutes: 5
  run: |
  cd /jfs/
  git clone https://github.com/danjacques/gofslock.git
  cd gofslock && git checkout cc7f001fe0e7df1710adc8f0cd9e9d6d21fdb3a9
  go test -v ./fslock/...
  stat /jfs/
 
-
- - name: Pyxattr Test
+ - name: flock test
+  timeout-minutes: 5
  run: |
- git clone https://github.com/iustin/pyxattr.git
- cd pyxattr
- pip3 install pytest
- pip3 install pyxattr
- stat /jfs/
- TEST_DIR=/jfs/ python3 -m pytest tests
+ git clone https://github.com/gofrs/flock.git
+ mkdir /jfs/tmp
+ cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test .
 
-
- - name: Fstests
+ - name: make secfs.test
  run: |
- sudo .github/scripts/apt_install.sh libacl1-dev attr
- sudo DURATION=60 make -C fstests fsx
- sudo make -C fstests xattrs
- sudo make -C fstests flock
-
-
- - name: Fsracer
- #https://github.com/gfx/example-github-actions-with-tty
- shell: 'script -q -e -c "bash {0}"'
+ sudo .github/scripts/apt_install.sh libacl1-dev
+ git clone https://github.com/billziss-gh/secfs.test.git
+ make -C secfs.test
+ 
+ - name: Fsx Test
+ timeout-minutes: 16
  run: |
- sudo DURATION=60 make -C fstests fsracer
-
- - name: MountJuiceFSTest
+ if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then
+ duration=900
+ else
+ duration=300
+ fi
+ sudo touch /jfs/fsx.out
+ sudo rm -f /tmp/fsx.out
+ sudo ln -s /jfs/fsx.out /tmp/fsx.out
+ sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out
+
+ - name: Fsracer Test
+ timeout-minutes: 16
+ shell: 'script -q -e -c "bash {0}"'
  run: |
- sudo /tmp/mount.juicefs redis://127.0.0.1:6379/1 /tmp/mount-jfs
- [ `stat --format "%i" /tmp/mount-jfs` -eq 1 ]
-
-
+ if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then
+ duration=900
+ else
+ duration=300
+ fi
+ sudo secfs.test/tools/bin/fsracer $duration /jfs
+ 
  - name: log
- if: ${{ always() }}
+ if: always()
  run: |
  tail -300 /var/log/juicefs.log
  grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
+ - name: Setup upterm session
+ if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+ timeout-minutes: 60
+ uses: lhotari/action-upterm@v1
+
+ success-all-test:
+ runs-on: ubuntu-latest
+ needs: [integrationtests]
+ if: always()
+ steps:
+ - uses: technote-space/workflow-conclusion-action@v3
+ - uses: actions/checkout@v3
+
+ - name: Check Failure
+ if: env.WORKFLOW_CONCLUSION == 'failure'
+ run: exit 1
+
  - name: Send Slack Notification
- if: ${{ failure() }}
+ if: failure() && github.event_name != 'workflow_dispatch'
  uses: juicedata/slack-notify-action@main
  with:
  channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
  slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
-
- - name: Setup upterm session
- if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
- timeout-minutes: 60
- uses: lhotari/action-upterm@v1
+ - name: Success
+ if: success() 
+ run: echo "All Done"

.github/workflows/resources/vdbench_big_file.conf

@@ -1,3 +1,4 @@
+data_errors=1
 fsd=fsd1,anchor=/tmp/vdbench/vdbench-big,depth=1,width=1,files=4,size=1g,openflags=o_direct
 
 fwd=fwd1,fsd=fsd1,operation=write,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4

.github/workflows/resources/vdbench_long_run.conf

@@ -0,0 +1,4 @@
+data_errors=1
+fsd=fsd1,anchor=/tmp/jfs,depth=1,width=2,files=2,sizes=(30m,0),shared=yes,openflags=o_direct
+fwd=fwd1,fsd=fsd1,threads=4,xfersize=(512,20,4k,20,64k,20,512k,20,1024k,20),fileio=random,fileselect=random,rdpct=70
+rd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=1800,interval=1

.github/workflows/resources/vdbench_small_file.conf

@@ -1,3 +1,4 @@
+data_errors=1
 fsd=fsd1,anchor=/tmp/vdbench/vdbench-small,depth=3,width=10,files=10,size=128k,openflags=o_direct
 
 fwd=fwd1,fsd=fsd1,operation=write,xfersize=128k,fileio=random,fileselect=random,threads=4