diff --git a/src/main/python/run_fusion_regression.py b/src/main/python/run_fusion_regression.py index bdca48a6c9..9c5a3c7ecd 100644 --- a/src/main/python/run_fusion_regression.py +++ b/src/main/python/run_fusion_regression.py @@ -59,7 +59,7 @@ def construct_fusion_commands(yaml_data: dict) -> list: return [ [ FUSE_COMMAND, - '-runs', ' '.join([run for run in yaml_data['runs']]), + '-runs', ' '.join(run['file'] for run in yaml_data['runs']), '-output', method.get('output'), '-method', method.get('name', 'average'), '-k', str(method.get('k', 1000)), @@ -166,6 +166,12 @@ def evaluate_and_verify(yaml_data: dict, dry_run: bool): logger.error(f"Failed to load configuration file: {e}") exit(1) + # Check existence of run files + for run in yaml_data['runs']: + if not os.path.exists(run['file']): + logger.error(f"Run file {run['file']} does not exist. Please run the dependent regressions first, recorded in the fusion yaml file.") + exit(1) + # Construct the fusion command fusion_commands = construct_fusion_commands(yaml_data) @@ -178,4 +184,4 @@ def evaluate_and_verify(yaml_data: dict, dry_run: bool): # Evaluate and verify results evaluate_and_verify(yaml_data, args.dry_run) - logger.info(f"Total execution time: {time.time() - start_time:.2f} seconds") + logger.info(f"Total execution time: {time.time() - start_time:.2f} seconds") \ No newline at end of file diff --git a/src/main/resources/fuse_regression/beir-v1.0.0-robust04.yaml b/src/main/resources/fuse_regression/beir-v1.0.0-robust04.yaml index d7051dc1f7..6fa234c785 100644 --- a/src/main/resources/fuse_regression/beir-v1.0.0-robust04.yaml +++ b/src/main/resources/fuse_regression/beir-v1.0.0-robust04.yaml @@ -32,10 +32,14 @@ topics: path: topics.beir-v1.0.0-robust04.test.tsv.gz qrel: qrels.beir-v1.0.0-robust04.test.txt -# Fusion Regression Test Configuration +# Run dependencies for fusion runs: - - runs/run.inverted.beir-v1.0.0-robust04.flat.test.bm25 - - runs/run.flat.beir-v1.0.0-robust04.bge-base-en-v1.5.test.bge-flat-onnx + - name: flat-bm25 + dependency: beir-v1.0.0-robust04.flat.yaml + file: runs/run.inverted.beir-v1.0.0-robust04.flat.test.bm25 + - name: bge-flat-onnx + dependency: beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml + file: runs/run.flat.beir-v1.0.0-robust04.bge-base-en-v1.5.test.bge-flat-cached methods: - name: rrf