Batch allows duplicate folder names in path

gaps/batch.py

@@ -104,10 +104,10 @@ def _make_job_dirs(self):
                 # Add the job tag to the directory path.
                 # This will copy config subdirs into the job subdirs
                 source_dir = Path(source_dir)
-                index = source_dir.parts.index(self._base_dir.name)
-                destination_dir = self._base_dir / tag
-                destination_dir = destination_dir.joinpath(
-                    *source_dir.parts[index + 1 :]
+                destination_dir = (
+                    self._base_dir
+                    / tag
+                    / source_dir.relative_to(self._base_dir)
                 )
                 destination_dir.mkdir(parents=True, exist_ok=True)
 
@@ -465,9 +465,13 @@ def _clean_arg(arg):
     if not isinstance(arg, str):
         return arg
 
-    missing_bracket_pair = "{" not in arg or "}" not in arg
     missing_string_quotes = "'" not in arg and '"' not in arg
-    if missing_bracket_pair or missing_string_quotes:
+    if missing_string_quotes:
+        return arg
+
+    missing_curly_bracket_pair = "{" not in arg or "}" not in arg
+    missing_bracket_pair = "[" not in arg or "]" not in arg
+    if missing_curly_bracket_pair and missing_bracket_pair:
         return arg
 
     arg = arg.replace("'", '"')

gaps/version.py

@@ -1,3 +1,3 @@
 """GAPs Version Number. """
 
-__version__ = "0.6.4"
+__version__ = "0.6.5"

tests/data/batch_project_1/config_batch.csv

@@ -1,5 +1,5 @@
 resolution,gen_fpath,econ_fpath,sc_features,trans_table,big_brown_bat,pipeline_config,set_tag,files
-75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd0/no_curtailment_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd0,"['./config_aggregation.json', './config_supply-curve.json']"
+75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd0/no_curtailment_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"[{'dset': 'big_brown_bat', 'method': 'sum'}, 'test', 0]",./config_pipeline.json,no_curtailment_sd0,"['./config_aggregation.json', './config_supply-curve.json']"
 115,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd1/no_curtailment_sd1_multi-year.h5,/projects/rev/data/transmission/build/multipliers_115.csv,/projects/rev/data/transmission/build/connections_115.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd1,"['./config_aggregation.json', './config_supply-curve.json']"
 165,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/no_curtailment/no_curtailment_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/no_curtailment_sd2/no_curtailment_sd2_multi-year.h5,/projects/rev/data/transmission/build/multipliers_165.csv,/projects/rev/data/transmission/build/connections_165.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,no_curtailment_sd2,"['./config_aggregation.json', './config_supply-curve.json']"
 75,/shared-projects/rev/projects/weto/bat_curtailment/rev_generation/blanket_dr0_ws5/blanket_dr0_ws5_multi-year.h5,/shared-projects/rev/projects/weto/bat_curtailment/rev_econ/blanket_cf0_sd0/blanket_cf0_sd0_multi-year.h5,/projects/rev/data/transmission/build/multipliers_075.csv,/projects/rev/data/transmission/build/connections_075.csv,"{'dset': 'big_brown_bat', 'method': 'sum'}",./config_pipeline.json,blanket_cf0_sd0,"['./config_aggregation.json', './config_supply-curve.json']"

tests/test_batch.py

@@ -34,12 +34,25 @@
 
 
 @pytest.fixture
-def typical_batch_config(test_data_dir, tmp_path):
+def typical_batch_config(test_data_dir, tmp_path, request):
     """All batch configs to be used in tests"""
 
     src_dir = test_data_dir / "batch_project_0"
-    shutil.copytree(src_dir, tmp_path / "batch_project_0")
-    return tmp_path / "batch_project_0" / "config_batch.json"
+    if request.param:
+        dst_dir = (
+            tmp_path
+            / "batch_project_0"
+            / "a"
+            / "b"
+            / "a"
+            / "c"
+            / "batch_project_0"
+        )
+        dst_dir.parent.mkdir(parents=True)
+    else:
+        dst_dir = tmp_path / "batch_project_0"
+    shutil.copytree(src_dir, dst_dir)
+    return dst_dir / "config_batch.json"
 
 
 @pytest.fixture
@@ -278,6 +291,7 @@ def test_invalid_mod_file_input(batch_config_with_yaml):
         BatchJob(bad_config_file).delete()
 
 
+@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True)
 def test_batch_job_setup(typical_batch_config, monkeypatch):
     """Test the creation and deletion of a batch job directory.
     Does not test batch execution which will require slurm."""
@@ -381,6 +395,7 @@ def _test_call(*cmd):
     assert count_2 == count_0, "Batch did not clear all job files!"
 
 
+@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True)
 def test_batch_job_run(typical_batch_config, monkeypatch):
     """Test a batch job run."""
 
@@ -541,11 +556,25 @@ def test_batch_csv_setup(csv_batch_config):
     assert arg["dset"] == "big_brown_bat"  # cspell: disable-line
     assert arg["method"] == "sum"
 
+    # test that the list was input properly
+    fp_agg = batch_dir / "no_curtailment_sd0" / "config_aggregation.json"
+    with open(fp_agg, "r") as config_file:
+        config_agg = json.load(config_file)
+    arg = config_agg["data_layers"]["big_brown_bat"]
+    assert isinstance(arg, list)
+    assert len(arg) == 3
+    assert isinstance(arg[0], dict)
+    assert arg[0]["dset"] == "big_brown_bat"  # cspell: disable-line
+    assert arg[0]["method"] == "sum"
+    assert arg[1] == "test"
+    assert arg[2] == 0
+
     BatchJob(csv_batch_config).delete()
     count_2 = len(list(batch_dir.glob("*")))
     assert count_2 == count_0, "Batch did not clear all job files!"
 
 
+@pytest.mark.parametrize("typical_batch_config", (True, False), indirect=True)
 def test_bad_str_arg(typical_batch_config):
     """Test that a string in a batch argument will raise an error (argument
     parameterizations should be lists)"""