.

tools/benchmarks/llm_eval_harness/meta_eval/eval_config.yaml

@@ -11,16 +11,16 @@ tensor_parallel_size: 1 # The VLLM argument that speicify the tensor parallel si
 
 data_parallel_size: 1 # The VLLM argument that speicify the data parallel size for the model, eg how copies of model will be used.
 
-gpu_memory_utilization: 0.9 #The VLLM argument that speicify gpu memory utilization, the rest will be reserved for KV cache.
+gpu_memory_utilization: 0.15 #The VLLM argument that speicify gpu memory utilization, the rest will be reserved for KV cache.
 
-max_model_len: 8192 #The VLLM argument that speicify model max length, decrease this value only if GPU memory issue encountered. Please make sure the max_gen_toks in the yaml does not exceed this length.
+max_model_len: 4000 #The VLLM argument that speicify model max length, decrease this value only if GPU memory issue encountered. Please make sure the max_gen_toks in the yaml does not exceed this length.
 
 batch_size: "auto" # Batch size, can be 'auto', 'auto:N', or an integer. It is strongly recommend to use 'auto' for vllm to speed up the inference
 
 output_path: "eval_results" # the output folder to store all the eval results and samples.
 
 #limit: 12 # Limit number of examples per task, set 'null' to run all.
-limit: 5 # Limit number of examples per task, set 'null' to run all.
+limit: 100 # Limit number of examples per task, set 'null' to run all.
 
 verbosity: "INFO" #Logging level: CRITICAL, ERROR, WARNING, INFO, DEBUG.
 

tools/benchmarks/llm_eval_harness/meta_eval/meta_template/meta_pretrain.yaml

@@ -1,4 +1,4 @@
 group: meta_pretrain
 task:
   - meta_bbh
-  - meta_mmlu_pro_pretrain
+  - meta_mmlu_pro_pretrain

tools/benchmarks/llm_eval_harness/meta_eval/meta_template/mmlu/utils.py

@@ -3,13 +3,14 @@
 
 def doc_to_text(doc: dict) -> str:
     # Strip out the last two characters, which is a space and the answer
+    # E.g., "Answer: B" -> "Answer:"
     return doc["input_final_prompts"][0][:-2]
 
 def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
     def _process_doc(doc: dict) -> dict:
-        # E.g., "Answer: B"
+        # input_correct_responses is in format of: "Answer: B"
         answer = doc["input_correct_responses"][0]
-        # Assumes that indexes are always A: 0, B: 1, C: 2, D: 3
+        # Indexes are always A: 0, B: 1, C: 2, D: 3
         answer_index = string.ascii_uppercase.index(answer[-1])
 
         out_doc = {