From 862996d1994451e176589d97acdc6fea979e3c8f Mon Sep 17 00:00:00 2001 From: bachvudinh Date: Sat, 24 Aug 2024 12:07:12 +0000 Subject: [PATCH] bump new version of AudioBench and add CI-CD test for ASR and SI benchmark --- .github/workflows/test-branch.yml | 38 +++++++++++++++++++++++++++--- .github/workflows/test-models.yml | 39 ++++++++++++++++++++++++++++--- AudioBench | 2 +- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-branch.yml b/.github/workflows/test-branch.yml index 760ffaa..54c4b98 100644 --- a/.github/workflows/test-branch.yml +++ b/.github/workflows/test-branch.yml @@ -26,8 +26,13 @@ on: required: false default: true type: boolean - run audio_benchmark: - description: 'Run audio benchmark test' + run_si_benchmark: + description: 'Run SI benchmark' + required: false + default: true + type: boolean + run_asr_benchmark: + description: 'Run ASR benchmark' required: false default: true type: boolean @@ -72,4 +77,31 @@ jobs: uses: actions/upload-artifact@v2 with: name: benchmark-results - path: ./lm-evaluation-harness/benchmark_results/*.json \ No newline at end of file + path: ./lm-evaluation-harness/benchmark_results/**/*.json + + - name: Eval on Speech Instruction Benchmark + if: ${{ github.event.inputs.run_si_benchmark == 'true' }} + env: + AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }} + run: | + cd AudioBench + pip3 install -r requirements.txt + chmod +x eval_si.sh + ./eval_si.sh ${{ github.event.inputs.model_id }} + + - name: Eval on ASR Benchmark + if: ${{ github.event.inputs.run_asr_benchmark == 'true' }} + env: + AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }} + run: | + cd AudioBench + pip3 install -r requirements.txt + chmod +x eval_asr.sh + ./eval_asr.sh ${{ github.event.inputs.model_id }} + + - name: Upload audio results + if: ${{ github.event.inputs.run_benchmark == 'true' }} + uses: actions/upload-artifact@v2 + with: + name: audio-benchmark-results + path: ./AudioBench/benchmark_results/log/**/*.json \ No newline at end of file diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml index 456c098..650a5db 100644 --- a/.github/workflows/test-models.yml +++ b/.github/workflows/test-models.yml @@ -22,11 +22,17 @@ on: required: false default: true type: boolean - run audio_benchmark: - description: 'Run audio benchmark test' + run_si_benchmark: + description: 'Run SI benchmark' required: false default: true type: boolean + run_asr_benchmark: + description: 'Run ASR benchmark' + required: false + default: true + type: boolean + jobs: run-test-and-benchmark: @@ -68,4 +74,31 @@ jobs: uses: actions/upload-artifact@v2 with: name: benchmark-results - path: ./lm-evaluation-harness/benchmark_results/**/*.json \ No newline at end of file + path: ./lm-evaluation-harness/benchmark_results/**/*.json + + - name: Eval on Speech Instruction Benchmark + if: ${{ github.event.inputs.run_si_benchmark == 'true' }} + env: + AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }} + run: | + cd AudioBench + pip3 install -r requirements.txt + chmod +x eval_si.sh + ./eval_si.sh ${{ github.event.inputs.model_id }} + + - name: Eval on ASR Benchmark + if: ${{ github.event.inputs.run_asr_benchmark == 'true' }} + env: + AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }} + run: | + cd AudioBench + pip3 install -r requirements.txt + chmod +x eval_asr.sh + ./eval_asr.sh ${{ github.event.inputs.model_id }} + + - name: Upload audio results + if: ${{ github.event.inputs.run_benchmark == 'true' }} + uses: actions/upload-artifact@v2 + with: + name: audio-benchmark-results + path: ./AudioBench/benchmark_results/log/**/*.json \ No newline at end of file diff --git a/AudioBench b/AudioBench index fa24fea..b2f1a60 160000 --- a/AudioBench +++ b/AudioBench @@ -1 +1 @@ -Subproject commit fa24fea1097c56be0c8c86b01ce37f3fc1f40267 +Subproject commit b2f1a60b57dc4a0a9f7a3654988c3c3829da6c02