new mllm eval

auto_round/__main__.py

@@ -41,11 +41,8 @@ def run_mllm():
         tune(args)
 
 def run_lmms():
-    try:
-        import importlib
-        importlib.import_module("lmms_eval")
-    except:
-        raise ImportError("please install the lmms_eval firt.")
+    from transformers.utils.versions import require_version
+    require_version("lmms_eval", "please install the lmms_eval firt.")
     # from auto_round.script.lmms_eval import setup_lmms_args, eval
     from auto_round.script.mllm import setup_lmms_parser, lmms_eval
     args = setup_lmms_parser()

auto_round/mllm/README.md

@@ -1,14 +1,13 @@
 # AutoRound for MLLMs
 ## Basic Usage (Gaudi2/CPU/GPU)
-A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the terminal.Alternatively, you can use ```auto_round``` instead of ```auto-round```. Set the format you want in `format` and
+A user guide detailing the full list of supported arguments is provided by calling ```auto-round-mllm -h``` on the terminal.Alternatively, you can use ```auto_round_mllm``` instead of ```auto-round-mllm```. Set the format you want in `format` and
 multiple formats exporting has been supported.
 
 ```bash
 auto—round-mllm \
     --model Qwen/Qwen2-VL-2B-Instruct\
     --bits 4 \
     --batch_size 1 \
-    --nsamples 128 \
     --gradient_accumulate_steps 4 \
     --group_size 128 \
     --format "auto_round" \
@@ -37,7 +36,7 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)
 ```
 
 ### Dataset
-For mllm, we used liuhaotian/llava_conv_58k as our default calib datasets. Through command ```--dataset```, user can use other datasets such as "liuhaotian/llava_instruct_80k", "liuhaotian/llava_instruct_150k" or a file path to use local file.
+For mllm, we used liuhaotian/llava_conv_58k as our default calib datasets. Through argument ```--dataset```, user can use other datasets such as "liuhaotian/llava_instruct_80k", "liuhaotian/llava_instruct_150k" or a file path to use local file.
 
 ### Limitation
 So far, auto-round for mllm supports five model families, include Qwen2, Llama, Phi3v, Llava and CogVLM2.

auto_round/mllm/eval.py

@@ -348,10 +348,7 @@ def lmms_eval(
         use_cache=None,
         apply_chat_template=False
         ):
-    try:
-        from auto_round import AutoRoundConfig
-    except:
-        from auto_round.auto_quantizer import AutoHfQuantizer
+    from auto_round import AutoRoundConfig
 
     if isinstance(tasks, str):
         tasks = tasks.replace(' ', '').split(',') 
@@ -379,6 +376,8 @@ def lmms_eval(
         model_args = f"model_id_name={model}"
     else:
         model_args = f"pretrained={model}"
+    if MODEL_TYPE_TO_LMMS_MODEL[model_type] == "llama_vision":
+        model_args += f",device_map={device}"
     results = _lmms_eval.evaluator.simple_evaluate(
         model=MODEL_TYPE_TO_LMMS_MODEL[model_type],
         model_args=model_args,