bump new version of AudioBench and add CI-CD test for ASR and SI benc…

…hmark
janhq · Aug 24, 2024 · 862996d · 862996d
1 parent ddac368
commit 862996d
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 7 deletions.
diff --git a/.github/workflows/test-branch.yml b/.github/workflows/test-branch.yml
@@ -26,8 +26,13 @@ on:
         required: false
         default: true
         type: boolean
-      run audio_benchmark:
-        description: 'Run audio benchmark test'
+      run_si_benchmark:
+        description: 'Run SI benchmark'
+        required: false
+        default: true
+        type: boolean
+      run_asr_benchmark:
+        description: 'Run ASR benchmark'
         required: false
         default: true
         type: boolean
@@ -72,4 +77,31 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: benchmark-results
-          path: ./lm-evaluation-harness/benchmark_results/*.json
+          path: ./lm-evaluation-harness/benchmark_results/**/*.json
+
+      - name: Eval on Speech Instruction Benchmark
+        if: ${{ github.event.inputs.run_si_benchmark == 'true' }}
+        env:
+          AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
+        run: |
+          cd AudioBench
+          pip3 install -r requirements.txt
+          chmod +x eval_si.sh
+          ./eval_si.sh  ${{ github.event.inputs.model_id }}
+
+      - name: Eval on ASR Benchmark
+        if: ${{ github.event.inputs.run_asr_benchmark == 'true' }}
+        env:
+          AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
+        run: |
+          cd AudioBench
+          pip3 install -r requirements.txt
+          chmod +x eval_asr.sh
+          ./eval_asr.sh  ${{ github.event.inputs.model_id }} 
+
+      - name: Upload audio results
+        if: ${{ github.event.inputs.run_benchmark == 'true' }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: audio-benchmark-results
+          path: ./AudioBench/benchmark_results/log/**/*.json
diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml
@@ -22,11 +22,17 @@ on:
         required: false
         default: true
         type: boolean
-      run audio_benchmark:
-        description: 'Run audio benchmark test'
+      run_si_benchmark:
+        description: 'Run SI benchmark'
         required: false
         default: true
         type: boolean
+      run_asr_benchmark:
+        description: 'Run ASR benchmark'
+        required: false
+        default: true
+        type: boolean
+
 
 jobs:
   run-test-and-benchmark:
@@ -68,4 +74,31 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: benchmark-results
-          path: ./lm-evaluation-harness/benchmark_results/**/*.json
+          path: ./lm-evaluation-harness/benchmark_results/**/*.json
+
+      - name: Eval on Speech Instruction Benchmark
+        if: ${{ github.event.inputs.run_si_benchmark == 'true' }}
+        env:
+          AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
+        run: |
+          cd AudioBench
+          pip3 install -r requirements.txt
+          chmod +x eval_si.sh
+          ./eval_si.sh  ${{ github.event.inputs.model_id }}
+
+      - name: Eval on ASR Benchmark
+        if: ${{ github.event.inputs.run_asr_benchmark == 'true' }}
+        env:
+          AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
+        run: |
+          cd AudioBench
+          pip3 install -r requirements.txt
+          chmod +x eval_asr.sh
+          ./eval_asr.sh  ${{ github.event.inputs.model_id }} 
+
+      - name: Upload audio results
+        if: ${{ github.event.inputs.run_benchmark == 'true' }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: audio-benchmark-results
+          path: ./AudioBench/benchmark_results/log/**/*.json
diff --git a/AudioBench b/AudioBench