diff --git a/gaps/batch.py b/gaps/batch.py index 8e516ff2..eb81b25a 100644 --- a/gaps/batch.py +++ b/gaps/batch.py @@ -104,10 +104,10 @@ def _make_job_dirs(self): # Add the job tag to the directory path. # This will copy config subdirs into the job subdirs source_dir = Path(source_dir) - index = source_dir.parts.index(self._base_dir.name) - destination_dir = self._base_dir / tag - destination_dir = destination_dir.joinpath( - *source_dir.parts[index + 1 :] + destination_dir = ( + self._base_dir + / tag + / source_dir.relative_to(self._base_dir) ) destination_dir.mkdir(parents=True, exist_ok=True) @@ -465,9 +465,13 @@ def _clean_arg(arg): if not isinstance(arg, str): return arg - missing_bracket_pair = "{" not in arg or "}" not in arg missing_string_quotes = "'" not in arg and '"' not in arg - if missing_bracket_pair or missing_string_quotes: + if missing_string_quotes: + return arg + + missing_curly_bracket_pair = "{" not in arg or "}" not in arg + missing_bracket_pair = "[" not in arg or "]" not in arg + if missing_curly_bracket_pair and missing_bracket_pair: return arg arg = arg.replace("'", '"') diff --git a/gaps/version.py b/gaps/version.py index 3018bf8b..70a7fc50 100644 --- a/gaps/version.py +++ b/gaps/version.py @@ -1,3 +1,3 @@ """GAPs Version Number. """ -__version__ = "0.6.4" +__version__ = "0.6.5" diff --git a/tests/data/batch_project_1/config_batch.csv b/tests/data/batch_project_1/config_batch.csv index 7cde1abc..54b795f2 100644 --- a/tests/data/batch_project_1/config_batch.csv +++ b/tests/data/batch_project_1/config_batch.csv @@ -1,5 +1,5 @@ resolution,gen_fpath,econ_fpath,sc_features,trans_table,big_brown_bat,pipeline_config,set_tag,files -75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd0/no_curtailment_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd0,"['./config_aggregation.json', './config_supply-curve.json']" +75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd0/no_curtailment_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"[{'dset': 'big_brown_bat', 'method': 'sum'}, 'test', 0]",./config_pipeline.json,no_curtailment_sd0,"['./config_aggregation.json', './config_supply-curve.json']" 115,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd1/no_curtailment_sd1_multi-year.h5,/projects/rev/data/transmission/build/multipliers_115.csv,/projects/rev/data/transmission/build/connections_115.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd1,"['./config_aggregation.json', './config_supply-curve.json']" 165,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd2/no_curtailment_sd2_multi-year.h5,/projects/rev/data/transmission/build/multipliers_165.csv,/projects/rev/data/transmission/build/connections_165.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd2,"['./config_aggregation.json', './config_supply-curve.json']" 75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/blanket_dr0_ws5/blanket_dr0_ws5_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/blanket_cf0_sd0/blanket_cf0_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,blanket_cf0_sd0,"['./config_aggregation.json', './config_supply-curve.json']" diff --git a/tests/test_batch.py b/tests/test_batch.py index a739e3d4..a29fe445 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -34,12 +34,25 @@ @pytest.fixture -def typical_batch_config(test_data_dir, tmp_path): +def typical_batch_config(test_data_dir, tmp_path, request): """All batch configs to be used in tests""" src_dir = test_data_dir / "batch_project_0" - shutil.copytree(src_dir, tmp_path / "batch_project_0") - return tmp_path / "batch_project_0" / "config_batch.json" + if request.param: + dst_dir = ( + tmp_path + / "batch_project_0" + / "a" + / "b" + / "a" + / "c" + / "batch_project_0" + ) + dst_dir.parent.mkdir(parents=True) + else: + dst_dir = tmp_path / "batch_project_0" + shutil.copytree(src_dir, dst_dir) + return dst_dir / "config_batch.json" @pytest.fixture @@ -278,6 +291,7 @@ def test_invalid_mod_file_input(batch_config_with_yaml): BatchJob(bad_config_file).delete() +@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True) def test_batch_job_setup(typical_batch_config, monkeypatch): """Test the creation and deletion of a batch job directory. Does not test batch execution which will require slurm.""" @@ -381,6 +395,7 @@ def _test_call(*cmd): assert count_2 == count_0, "Batch did not clear all job files!" +@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True) def test_batch_job_run(typical_batch_config, monkeypatch): """Test a batch job run.""" @@ -541,11 +556,25 @@ def test_batch_csv_setup(csv_batch_config): assert arg["dset"] == "big_brown_bat" # cspell: disable-line assert arg["method"] == "sum" + # test that the list was input properly + fp_agg = batch_dir / "no_curtailment_sd0" / "config_aggregation.json" + with open(fp_agg, "r") as config_file: + config_agg = json.load(config_file) + arg = config_agg["data_layers"]["big_brown_bat"] + assert isinstance(arg, list) + assert len(arg) == 3 + assert isinstance(arg[0], dict) + assert arg[0]["dset"] == "big_brown_bat" # cspell: disable-line + assert arg[0]["method"] == "sum" + assert arg[1] == "test" + assert arg[2] == 0 + BatchJob(csv_batch_config).delete() count_2 = len(list(batch_dir.glob("*"))) assert count_2 == count_0, "Batch did not clear all job files!" +@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True) def test_bad_str_arg(typical_batch_config): """Test that a string in a batch argument will raise an error (argument parameterizations should be lists)"""