Fix custom model wrapper to enable usage of instanced model (#508)

* fix to simple_evaluate method * fix to simple_evaluate method * fix
EvolvingLMMs-Lab · Jan 23, 2025 · f6fe367 · f6fe367
1 parent 9c4929f
commit f6fe367
Showing 1 changed file with 13 additions and 8 deletions.
diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py
@@ -170,14 +170,19 @@ def simple_evaluate(
 
     task_dict = get_task_dict(tasks, task_manager)
 
-    ModelClass = get_model(model)
-    lm = ModelClass.create_from_arg_string(
-        model_args,
-        {
-            "batch_size": batch_size,
-            "device": device,
-        },
-    )
+    if isinstance(model, str):
+        if model_args is None:
+            model_args = ""
+        lm = lmms_eval.models.get_model(model).create_from_arg_string(
+            model_args,
+            {
+                "batch_size": batch_size,
+                "max_batch_size": max_batch_size,
+                "device": device,
+            },
+        )
+    elif isinstance(model, lmms_eval.api.model.lmms):
+        lm = model
 
     # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
     # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)