fix slurm scripts

Author: wangshangsam

multimodal/vl2l/scripts/slurm/benchmark.sh

@@ -25,8 +25,6 @@ srun \
         --vllm.model.repo_id=${MODEL_REPO_ID} \
         --vllm.cli=--async-scheduling \
         --vllm.cli=--max-model-len=32768 \
-        --vllm.cli=--max-num-seqs=1024 \
-        --vllm.cli=--mm-encoder-tp-mode=data \
         --vllm.cli=--limit-mm-per-prompt.video=0 \
         --vllm.cli=--tensor-parallel-size=8 \
         --settings.logging.log_output.outdir=${OUTPUT_CONTAINER_DIR}/${SLURM_JOB_ID} 

multimodal/vl2l/scripts/slurm/evaluate.sh

@@ -9,10 +9,13 @@
 #SBATCH --output=evaluate-slurm-output-%j.txt
 #SBATCH --error=evaluate-slurm-error-%j.txt
 
+export NVIDIA_VISIBLE_DEVICES=void
+
 srun \
     --container-image=${CONTAINER_IMAGE} \
     --container-mounts=${CACHE_HOST_DIR}:${CACHE_CONTAINER_DIR},${OUTPUT_HOST_DIR}:${OUTPUT_CONTAINER_DIR} \
     --no-container-mount-home \
+    --container-env=NVIDIA_VISIBLE_DEVICES \
     mlperf-inf-mm-vl2l evaluate \
         --dataset.token=${DATASET_TOKEN} \
-        --filename=${OUTPUT_CONTAINER_DIR}/${SLURM_JOB_ID}/mlperf_log_accuracy.json
+        --filename=${OUTPUT_CONTAINER_DIR}/${BENCHMARK_JOB_ID}/mlperf_log_accuracy.json

multimodal/vl2l/scripts/slurm/submit.sh

@@ -197,7 +197,7 @@ benchmark_job_id=$(
 if [[ "${mode}" == "accuracy_only" ]]; then
     sbatch \
         --dependency=afterok:"${benchmark_job_id}" \
-        --export=CACHE_HOST_DIR="${cache_host_dir}",CACHE_CONTAINER_DIR="${cache_container_dir}",OUTPUT_HOST_DIR="${output_host_dir}",OUTPUT_CONTAINER_DIR="${output_container_dir}",CONTAINER_IMAGE="${container_image}",DATASET_TOKEN="${dataset_token}" \
+        --export=CACHE_HOST_DIR="${cache_host_dir}",CACHE_CONTAINER_DIR="${cache_container_dir}",OUTPUT_HOST_DIR="${output_host_dir}",OUTPUT_CONTAINER_DIR="${output_container_dir}",CONTAINER_IMAGE="${container_image}",DATASET_TOKEN="${dataset_token}",BENCHMARK_JOB_ID="${benchmark_job_id}" \
         --account="${slurm_account}" \
         --partition="${evaluate_slurm_partition}" \
         evaluate.sh