From 7102906ea0a1becab4c2271f64c36fa596bf00b0 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 18 Dec 2023 14:34:11 -0700 Subject: [PATCH 1/3] Logging now default to info when using CSV batch file --- gaps/cli/batch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gaps/cli/batch.py b/gaps/cli/batch.py index 8a6171e6..b316705a 100644 --- a/gaps/cli/batch.py +++ b/gaps/cli/batch.py @@ -5,6 +5,7 @@ import click import gaps.batch +from gaps.log import init_logger from gaps.config import init_logging_from_config_file from gaps.cli.command import _WrappedCommand from gaps.cli.documentation import _batch_command_help @@ -12,7 +13,12 @@ def _batch(config_file, dry, cancel, delete, monitor_background): """Execute an analysis pipeline over a parametric set of inputs""" - init_logging_from_config_file(config_file, background=monitor_background) + if str(config_file).endswith("csv"): + init_logger(stream=not monitor_background, level="INFO", file=None) + else: + init_logging_from_config_file( + config_file, background=monitor_background + ) if cancel: gaps.batch.BatchJob(config_file).cancel() From c5eeb2faff53e63e0ab445eb6c3834bf0c72d958 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 18 Dec 2023 14:34:21 -0700 Subject: [PATCH 2/3] Bump version --- gaps/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gaps/version.py b/gaps/version.py index 97748d9b..3018bf8b 100644 --- a/gaps/version.py +++ b/gaps/version.py @@ -1,3 +1,3 @@ """GAPs Version Number. """ -__version__ = "0.6.3" +__version__ = "0.6.4" From f60951d0186252254b3ecd5971252b56b242ae04 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 18 Dec 2023 14:49:40 -0700 Subject: [PATCH 3/3] Add test for batch CSV from CLI --- tests/cli/test_cli_batch.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/cli/test_cli_batch.py b/tests/cli/test_cli_batch.py index 6e0dba63..66bf87dc 100644 --- a/tests/cli/test_cli_batch.py +++ b/tests/cli/test_cli_batch.py @@ -4,9 +4,12 @@ GAPs batch command tests. """ import json +import shutil from pathlib import Path import pytest +import pandas as pd +from pandas.testing import assert_frame_equal from gaps.batch import BatchJob from gaps.cli.batch import batch_command @@ -77,5 +80,52 @@ def _cache_args_kwargs(self, dry_run, monitor_background): assert arg_cache[-1][1] +# pylint: disable=too-many-locals +def test_batch_csv(test_data_dir, tmp_path, cli_runner): + """Test a batch project setup from csv config""" + + src_dir = test_data_dir / "batch_project_1" + batch_dir = tmp_path / "batch_project_1" + shutil.copytree(src_dir, batch_dir) + csv_batch_config = batch_dir / "config_batch.csv" + + config_table = pd.read_csv(csv_batch_config) + count_0 = len(list(batch_dir.glob("*"))) + assert count_0 == 5, "Unknown starting files detected!" + + bc = batch_command() + cli_runner.invoke(bc, ["-c", csv_batch_config, "--dry"]) + + dirs = set(fp.name for fp in batch_dir.glob("*")) + count_1 = len(dirs) + assert (count_1 - count_0) == len(config_table) + 1 + + job_table = pd.read_csv(batch_dir / "batch_jobs.csv", index_col=0) + for job in job_table.index.values: + assert job in dirs + + job_table.index.name = "index" + compare_cols = set(config_table.columns) + compare_cols -= {"pipeline_config", "files"} + compare_cols = list(compare_cols) + assert_frame_equal( + config_table[compare_cols].reset_index(drop=True), + job_table[compare_cols].reset_index(drop=True), + ) + + # test that the dict was input properly + fp_agg = batch_dir / "blanket_cf0_sd0" / "config_aggregation.json" + with open(fp_agg, "r") as config_file: + config_agg = json.load(config_file) + arg = config_agg["data_layers"]["big_brown_bat"] + assert isinstance(arg, dict) + assert arg["dset"] == "big_brown_bat" # cspell: disable-line + assert arg["method"] == "sum" + + cli_runner.invoke(bc, ["-c", csv_batch_config, "--delete"]) + count_2 = len(list(batch_dir.glob("*"))) + assert count_2 == count_0, "Batch did not clear all job files!" + + if __name__ == "__main__": pytest.main(["-q", "--show-capture=all", Path(__file__), "-rapP"])