Merge pull request NVlabs#67 from Efficient-Large-Model/dev/fix_shard…

…ed_eval Fix bugs in sharded evaluation scripts
gheinrich · May 1, 2024 · 4f906a9 · 4f906a9
2 parents 2f66927 + fd01d2c
commit 4f906a9
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/scripts/v1_5/eval/mmbench_cn_sharded.sh b/scripts/v1_5/eval/mmbench_cn_sharded.sh
@@ -21,7 +21,7 @@ for IDX in $(seq 0 $((CHUNKS-1))); do
     CUDA_VISIBLE_DEVICES=${GPULIST[$GPU_IDX1]},${GPULIST[$GPU_IDX2]} python -m llava.eval.model_vqa_mmbench \
         --model-path $MODEL_PATH \
         --question-file ./playground/data/eval/mmbench_cn/$SPLIT.tsv \
-        --answers-file ./eval_output/$CKPT/mmbench_cn/$SPLIT.jsonl \
+        --answers-file ./eval_output/$CKPT/mmbench_cn/${CHUNKS}_${IDX}.jsonl \
         --lang cn \
         --single-pred-prompt \
         --temperature 0 \

diff --git a/scripts/v1_5/eval/seed_sharded.sh b/scripts/v1_5/eval/seed_sharded.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
 MODEL_PATH=$1
 
 gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
@@ -44,4 +45,4 @@ done
 python scripts/convert_seed_for_submission.py \
     --annotation-file ./playground/data/eval/seed_bench/SEED-Bench.json \
     --result-file $output_file \
-    --result-upload-file ./eval_output/$CKPT/seed_bench/answers_upload.jsonl
+    --result-upload-file ./eval_output/$CKPT/seed_bench/answers_upload.jsonl