hadoop: avoid create too many threads for multiple JuiceFileSystem instance #6152
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "pjdfstest" | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release-**' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| pull_request: | |
| #The branches below must be a subset of the branches above | |
| branches: | |
| - 'main' | |
| - 'release-**' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| - '.github/**' | |
| schedule: | |
| - cron: '0 19 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| type: boolean | |
| description: "Run the build with tmate debugging enabled" | |
| required: false | |
| default: false | |
| jobs: | |
| build-matrix: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| echo "github.event_name is ${{github.event_name}}" | |
| echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}" | |
| if [ "${{github.event_name}}" == "schedule" ]; then | |
| echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT | |
| elif [ "${{github.event_name}}" == "pull_request" ]; then | |
| echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT | |
| elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then | |
| echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT | |
| else | |
| echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT | |
| fi | |
| outputs: | |
| meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} | |
| pjdfstest: | |
| needs: build-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'badger', 'mariadb', 'fdb'] | |
| meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: shogo82148/actions-setup-perl@v1 | |
| with: | |
| perl-version: '5.34' | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set Variable | |
| id: vars | |
| run: | | |
| if [ "${{matrix.meta}}" == "fdb" ]; then | |
| echo "target=juicefs.fdb" >> $GITHUB_OUTPUT | |
| else | |
| echo "target=juicefs" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build | |
| uses: ./.github/actions/build | |
| with: | |
| target: ${{steps.vars.outputs.target}} | |
| - name: Prepare meta db | |
| run: | | |
| chmod +x .github/scripts/start_meta_engine.sh | |
| source .github/scripts/start_meta_engine.sh | |
| start_meta_engine ${{matrix.meta}} | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| create_database $meta_url | |
| - name: Juicefs Format | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo ./juicefs format $meta_url --trash-days 0 pics | |
| - name: Juicefs Mount | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| # sudo mkdir /var/jfs | |
| # sudo chmod 777 /var/jfs | |
| sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report & | |
| sleep 5 | |
| if [ ! -f /tmp/jfs/.accesslog ]; then | |
| echo "<FATAL>: mount failed" | |
| exit 1 | |
| fi | |
| - name: Pjdfstest | |
| run: | | |
| sudo .github/scripts/apt_install.sh libtap-harness-archive-perl | |
| cd /tmp/jfs/ | |
| git clone https://github.com/sanwan/pjdfstest.git | |
| cd pjdfstest | |
| autoreconf -ifs | |
| ./configure | |
| make pjdfstest | |
| sudo prove -rv tests/ | |
| - name: log | |
| if: ${{ always() }} | |
| shell: bash | |
| run: | | |
| if [ -f ~/.juicefs/juicefs.log ]; then | |
| tail -300 ~/.juicefs/juicefs.log | |
| grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true | |
| fi | |
| if [ -f /var/log/juicefs.log ]; then | |
| tail -300 /var/log/juicefs.log | |
| grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true | |
| fi | |
| - name: Send Slack Notification | |
| if: ${{ failure() }} | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Setup upterm session | |
| if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }} | |
| timeout-minutes: 60 | |
| uses: lhotari/action-upterm@v1 | |
| success-all-test: | |
| runs-on: ubuntu-latest | |
| needs: [pjdfstest] | |
| if: always() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v3 | |
| - uses: actions/checkout@v3 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Send Slack Notification | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Success | |
| if: ${{ success() }} | |
| run: echo "All Done" |