Add dependency handling

E3SM-Project · Dec 10, 2024 · e3e48cc · e3e48cc
1 parent 265d67a
commit e3e48cc
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 5 deletions.
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
@@ -7,7 +7,7 @@
 from mache import MachineInfo
 from PIL import Image, ImageChops, ImageDraw
 
-UNIQUE_ID = "unique_id"
+UNIQUE_ID = "test-650-20241210"
 
 # Image checking ##########################################################
 
@@ -146,12 +146,12 @@ def get_chyrsalis_expansions(config):
         "constraint": "",
         # To run this test, replace conda environment with your e3sm_diags dev environment
         # To use default environment_commands, set to ""
-        "diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
+        "diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate e3sm_diags_main_20241210",
         "diags_walltime": "5:00:00",
         "e3sm_to_cmip_environment_commands": "",
         "environment_commands_test": "",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
-        "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
+        "global_time_series_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi_dev_weekly_20241122",
         "mpas_analysis_walltime": "00:30:00",
         "partition_long": "compute",
         "partition_short": "debug",

diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py
@@ -285,6 +285,7 @@ def add_ts_dependencies(
     )
     depend_on_ts: Set[str] = set(["enso_diags", "qbo", "area_mean_time_series"])
     if depend_on_ts & set(c["sets"]):
+        # ts task
         add_dependencies(
             dependencies,
             script_dir,
@@ -294,6 +295,16 @@ def add_ts_dependencies(
             end_yr,
             c["ts_num_years"],
         )
+        # e3sm_to_cmip task
+        add_dependencies(
+            dependencies,
+            script_dir,
+            "e3sm_to_cmip",
+            ts_sub,  # Relies on having the same subsection name as the corresponding ts subsection!
+            start_yr,
+            end_yr,
+            c["ts_num_years"],
+        )
     if "streamflow" in c["sets"]:
         add_dependencies(
             dependencies,

diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py
@@ -65,7 +65,7 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f
                 dependencies,
                 script_dir,
                 "ts",
-                sub,
+                sub,  # Relies on having the same subsection name as the corresponding ts subsection!
                 c["yr_start"],
                 c["yr_end"],
                 c["ts_num_years"],

diff --git a/zppy/ilamb.py b/zppy/ilamb.py
@@ -109,6 +109,17 @@ def determine_and_add_dependencies(
         c["year2"],
         c["ts_num_years"],
     )
+    add_dependencies(
+        dependencies,
+        script_dir,
+        "e3sm_to_cmip",
+        c[
+            "ts_land_subsection"
+        ],  # Relies on having the same subsection name as the corresponding ts subsection!
+        c["year1"],
+        c["year2"],
+        c["ts_num_years"],
+    )
     if not c["land_only"]:
         define_or_guess2(
             c,
@@ -125,3 +136,14 @@ def determine_and_add_dependencies(
             c["year2"],
             c["ts_num_years"],
         )
+        add_dependencies(
+            dependencies,
+            script_dir,
+            "e3sm_to_cmip",
+            c[
+                "ts_atm_subsection"
+            ],  # Relies on having the same subsection name as the corresponding ts subsection!
+            c["year1"],
+            c["year2"],
+            c["ts_num_years"],
+        )
diff --git a/zppy/templates/e3sm_to_cmip.bash b/zppy/templates/e3sm_to_cmip.bash
@@ -4,7 +4,9 @@
 {{ environment_commands }}
 
 # Create temporary workdir
-workdir=`mktemp -d tmp.${id}.XXXX`
+hash=`mktemp --dry-run -d XXXX`
+workdir=tmp.{{ prefix }}.${id}.${hash}
+mkdir ${workdir}
 cd ${workdir}
 
 tmp_dir=tmp_{{ prefix }}