[Update] Update LiveMathBench Hard Configs

opencompass/configs/datasets/livemathbench/livemathbench_gen_9befbf.py

@@ -5,47 +5,54 @@
 from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
 
 
-livemathbench_dataset = dict(
-    type=LiveMathBenchDataset,
-    path='',
-    k=16,
-    replication=3,
-    dataset_splits=['CNMO', 'CCEE', 'AMC', 'WLPMC'],
-    dataset_languages=['cn', 'en'],
-    cot=True,
-    version='202412',
-    abbr='LiveMathBench-v202412',
-    reader_cfg=dict(
-        input_columns=['prompt'], 
-        output_column='answer'
+reader_cfg = dict(
+    input_columns=['prompt'], 
+    output_column='answer'
+)
+
+infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+            ]
+        )
     ),
-    infer_cfg=dict(
-        prompt_template=dict(
-            type=PromptTemplate,
-            template=dict(
-                round=[
-                    dict(role='HUMAN', prompt='{prompt}'),
-                ]
-            )
-        ),
-        retriever=dict(type=ZeroRetriever),
-        inferencer=dict(
-            type=GenInferencer, 
-            max_out_len=8192
-        ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(
+        type=GenInferencer, 
+        max_out_len=8192
     ),
-    eval_cfg=dict(
-        evaluator=dict(
-            type=LiveMathBenchEvaluator,
-            model_name='',
-            url=[],
-            use_extract_model=False,
-            extract_url=[],
-            extract_model_name='',
-            k=[4, 8, 16],
-            replication=3,
-            thresholds=[0.0, 0.25, 0.5, 0.75, 1.0]
-        )
+)
+
+eval_cfg = dict(
+    evaluator=dict(
+        type=LiveMathBenchEvaluator,
+        model_name='',
+        url=[],
+        use_extract_model=False,
+        extract_url=[],
+        extract_model_name='',
+        k=[4, 8, 16],
+        replication=3,
+        thresholds=[0.0, 0.25, 0.5, 0.75, 1.0]
     )
 )
-livemathbench_datasets = [livemathbench_dataset]
+
+livemathbench_datasets = [
+    dict(
+        type=LiveMathBenchDataset,
+        path='',
+        k=16,
+        replication=3,
+        dataset_splits=['CNMO', 'CCEE', 'AMC', 'WLPMC'],
+        dataset_languages=['cn', 'en'],
+        cot=True,
+        version='202412',
+        abbr='LiveMathBench-v202412-k16r3',
+        reader_cfg=reader_cfg,
+        infer_cfg=infer_cfg,
+        eval_cfg=eval_cfg
+    )
+]

opencompass/configs/datasets/livemathbench/livemathbench_gen_f1c095.py → opencompass/configs/datasets/livemathbench/livemathbench_greedy_gen_9befbf.py

@@ -4,13 +4,12 @@
 
 from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
 
-
-livemathbench_reader_cfg = dict(
+reader_cfg = dict(
     input_columns=['prompt'], 
     output_column='answer'
 )
 
-livemathbench_infer_cfg = dict(
+infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template=dict(
@@ -22,28 +21,34 @@
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(
         type=GenInferencer, 
-        max_out_len=8192,
-        temperature=1.0
-    )
+        max_out_len=8192
+    ),
 )
 
-livemathbench_eval_cfg = dict(
+eval_cfg = dict(
     evaluator=dict(
-        type=LiveMathBenchEvaluator, 
-        model_name='Qwen/Qwen2.5-72B-Instruct', 
-        url=['http://172.30.40.154:23333/v1/'] #'https://api.openai.com/v1/'
+        type=LiveMathBenchEvaluator,
+        model_name='',
+        url=[],
+        k=[1],
+        replication=1,
+        thresholds=[1.0]
     )
 )
 
 livemathbench_datasets = [
     dict(
         type=LiveMathBenchDataset,
-        abbr='LiveMathBench-k1-n1',
-        path='opencompass/LiveMathBench202412',
-        k=1, # K@Pass
-        n=1,  # Run times
-        reader_cfg=livemathbench_reader_cfg,
-        infer_cfg=livemathbench_infer_cfg,
-        eval_cfg=livemathbench_eval_cfg
+        path='',
+        k=1,
+        replication=1,
+        dataset_splits=['CNMO', 'CCEE', 'AMC', 'WLPMC'],
+        dataset_languages=['cn', 'en'],
+        cot=True,
+        version='202412',
+        abbr='LiveMathBench-v202412-k1r1',
+        reader_cfg=reader_cfg,
+        infer_cfg=infer_cfg,
+        eval_cfg=eval_cfg
     )
 ]

opencompass/configs/datasets/livemathbench/livemathbench_hard_gen_9befbf.py

@@ -0,0 +1,58 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+
+from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
+
+
+reader_cfg = dict(
+    input_columns=['prompt'], 
+    output_column='answer'
+)
+
+infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(
+        type=GenInferencer, 
+        max_out_len=8192
+    ),
+)
+
+eval_cfg = dict(
+    evaluator=dict(
+        type=LiveMathBenchEvaluator,
+        model_name='',
+        url=[],
+        use_extract_model=False,
+        extract_url=[],
+        extract_model_name='',
+        k=[4, 8, 16],
+        replication=3,
+        thresholds=[0.0, 0.25, 0.5, 0.75, 1.0]
+    )
+)
+
+livemathbench_datasets = [
+    dict(
+        type=LiveMathBenchDataset,
+        path='',
+        k=16,
+        replication=3,
+        dataset_splits=['hard'],
+        dataset_languages=['cn', 'en'],
+        cot=True,
+        version='202412',
+        abbr='LiveMathBench-v202412-hard-k16r3',
+        reader_cfg=reader_cfg,
+        infer_cfg=infer_cfg,
+        eval_cfg=eval_cfg
+    )
+]

opencompass/configs/datasets/livemathbench/livemathbench_hard_greedy_gen_9befbf.py

@@ -0,0 +1,58 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+
+from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
+
+
+reader_cfg = dict(
+    input_columns=['prompt'], 
+    output_column='answer'
+)
+
+infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(
+        type=GenInferencer, 
+        max_out_len=8192
+    ),
+)
+
+eval_cfg = dict(
+    evaluator=dict(
+        type=LiveMathBenchEvaluator,
+        model_name='',
+        url=[],
+        use_extract_model=False,
+        extract_url=[],
+        extract_model_name='',
+        k=[4, 8, 16],
+        replication=3,
+        thresholds=[0.0, 0.25, 0.5, 0.75, 1.0]
+    )
+)
+
+livemathbench_datasets = [
+    dict(
+        type=LiveMathBenchDataset,
+        path='',
+        k=16,
+        replication=3,
+        dataset_splits=['hard'],
+        dataset_languages=['cn', 'en'],
+        cot=True,
+        version='202412',
+        abbr='LiveMathBench-v202412-hard-k16r3',
+        reader_cfg=reader_cfg,
+        infer_cfg=infer_cfg,
+        eval_cfg=eval_cfg
+    )
+]

opencompass/datasets/livemathbench/livemathbench.py

@@ -48,6 +48,7 @@ def load(path: str,
         if path != '':
             path = get_data_path(path)
             path = os.path.join(path, version)
+
         for split, language in product(dataset_splits, dataset_languages):
             dataset_info[f'{split}_{language}'] = {
                 'single-choice': 0,
@@ -64,7 +65,6 @@ def load(path: str,
 
             if path != '':
                 file_path = os.path.join(path, f'{split}_{language}.jsonl')
-
                 if not os.path.exists(file_path):
                     raise FileNotFoundError(
                         f'File {file_path} does not exist, please check the '

opencompass/models/turbomind_with_tf_above_v4_33.py

@@ -164,8 +164,6 @@ def generate(self,
         self.logger.info('Generation Config of LMdeploy: ')
         self.logger.info(gen_config)
 
-
-
         results = []
         outputs = self.pipe(messages, gen_config=gen_config, do_preprocess=False)
         for output in outputs: