Native EAMxx support (#424)

* Native EAMxx support Initial set of changes for ts and climo tasks to accept native monthly EAMxx files as input. Supported by new version of ncclimo. * Revert use of custom ncclimo executable. * fix pre-committing error * Testing fixes * Integration test fixes * Rerun utils --------- Co-authored-by: chengzhuzhang <zhang40@llnl.gov> Co-authored-by: Ryan Forsyth <forsyth2@llnl.gov>
E3SM-Project · Feb 15, 2024 · a902549 · a902549
1 parent 3e796d3
commit a902549
Show file tree

Hide file tree

Showing 12 changed files with 58 additions and 26 deletions.
diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg
@@ -63,7 +63,7 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,LAISHA,LAISUN,RH2M"
+  vars = "FSH,RH2M"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg
@@ -55,7 +55,7 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,LAISHA,LAISUN,RH2M"
+  vars = "FSH,RH2M"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]
@@ -93,7 +93,7 @@ years = "1850:1854:2", "1850:1854:4",
   walltime = "5:00:00"
 
   [[ atm_monthly_180x360_aave_environment_commands ]]
-  environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20231027v2"
+  environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214"
   sets = "qbo",
   ts_subsection = "atm_monthly_180x360_aave"
 

diff --git a/tests/integration/template_bundles.cfg b/tests/integration/template_bundles.cfg
@@ -63,7 +63,7 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,LAISHA,LAISUN,RH2M"
+  vars = "FSH,RH2M"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/template_complete_run.cfg b/tests/integration/template_complete_run.cfg
@@ -55,7 +55,7 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,LAISHA,LAISUN,RH2M"
+  vars = "FSH,RH2M"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]

diff --git a/tests/integration/utils.py b/tests/integration/utils.py
@@ -136,7 +136,7 @@ def get_chyrsalis_expansions(config):
         "bundles_walltime": "07:00:00",
         "constraint": "",
         # To run this test, replace conda environment with your e3sm_diags dev environment
-        "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20231027v2",
+        "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214",
         "diags_walltime": "5:00:00",
         "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.9.0rc16_chrysalis.sh",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",

diff --git a/tests/test_sections.py b/tests/test_sections.py
@@ -101,6 +101,7 @@ def test_sections(self):
             "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "dpf": 30,
             "extra_vars": "",
+            "input_component": "",
             "mapping_file": "MAPPING_FILE_TS",
             "tpd": 1,
             "ts_fmt": "ts_only",
@@ -128,6 +129,7 @@ def test_sections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_TS",
@@ -154,6 +156,7 @@ def test_sections(self):
         expected_section = {
             "active": "True",
             "exclude": False,
+            "input_component": "",
             "mapping_file": "MAPPING_FILE_CLIMO",
             "nodes": 4,
             "parallel": "mpi",
@@ -178,6 +181,7 @@ def test_sections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_CLIMO",
@@ -254,13 +258,15 @@ def test_subsections(self):
             "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "dpf": 30,
             "extra_vars": "",
+            "input_component": "",
             "tpd": 1,
             "ts_fmt": "ts_only",
             "ts_grid1": {
                 "area_nm": None,
                 "cmip_metadata": None,
                 "dpf": None,
                 "extra_vars": None,
+                "input_component": None,
                 "mapping_file": "MAPPING_FILE_TS_GRID1",
                 "tpd": None,
                 "ts_fmt": None,
@@ -271,6 +277,7 @@ def test_subsections(self):
                 "cmip_metadata": None,
                 "dpf": None,
                 "extra_vars": None,
+                "input_component": None,
                 "mapping_file": "MAPPING_FILE_TS_GRID2",
                 "tpd": None,
                 "ts_fmt": None,
@@ -299,6 +306,7 @@ def test_subsections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_TS_GRID1",
@@ -336,6 +344,7 @@ def test_subsections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_TS_GRID2",
@@ -362,12 +371,14 @@ def test_subsections(self):
         expected_section = {
             "active": "True",
             "climo_grid1": {
+                "input_component": None,
                 "mapping_file": "MAPPING_FILE_CLIMO_GRID1",
                 "nodes": None,
                 "exclude": None,
                 "vars": None,
             },
             "climo_grid2": {
+                "input_component": None,
                 "mapping_file": "MAPPING_FILE_CLIMO_GRID2",
                 "years": ["0001:0100:50"],
                 "partition": "LONG",
@@ -376,6 +387,7 @@ def test_subsections(self):
                 "vars": None,
             },
             "exclude": False,
+            "input_component": "",
             "mapping_file": "MAPPING_FILE_CLIMO",
             "nodes": 4,
             "parallel": "mpi",
@@ -400,6 +412,7 @@ def test_subsections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_CLIMO_GRID1",
@@ -432,6 +445,7 @@ def test_subsections(self):
             "frequency": "monthly",
             "grid": "",
             "input": "INPUT",
+            "input_component": "",
             "input_files": "eam.h0",
             "input_subdir": "INPUT_SUBDIR",
             "mapping_file": "MAPPING_FILE_CLIMO_GRID2",

diff --git a/zppy/climo.py b/zppy/climo.py
@@ -55,8 +55,10 @@ def climo(config, scriptDir, existing_bundles, job_ids_file):
                         % (c["mapping_file"])
                     )
 
-        # Component
-        c["component"] = getComponent(c["input_files"])
+        # Output component (for directory structure) and procedure type for ncclimo
+        c["component"], c["prc_typ"] = getComponent(
+            c["input_component"], c["input_files"]
+        )
 
         # Loop over year sets
         year_sets = getYears(c["years"])

diff --git a/zppy/templates/climo.bash b/zppy/templates/climo.bash
@@ -25,7 +25,12 @@ cd ${workdir}
 {% if frequency == 'monthly' %}
 # --- Monthly climatologies ---
 ncclimo \
+{% if prc_typ == 'eamxx' -%}
+--case={{ case }}.{{ input_files }}.0001-01-01-00000.nc \
+--fml_nm={{ case }} \
+{%- else -%}
 --case={{ case }} \
+{%- endif %}
 --jobs=${SLURM_NNODES} \
 --thr=1 \
 {%- if exclude %}
@@ -47,11 +52,7 @@ ncclimo \
 --output=trash \
 --regrid=output \
 {%- endif %}
-{%- if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' or input_files.split(".")[0] == 'elm' or input_files.split(".")[0] == 'clm2'  %}
---prc_typ={{ input_files.split(".")[0] }}
-{%- else -%}
---prc_typ=sgs
-{%- endif %}
+--prc_typ={{ prc_typ }}
 
 {% elif frequency.startswith('diurnal') %}
 

diff --git a/zppy/templates/default.ini b/zppy/templates/default.ini
@@ -78,11 +78,14 @@ parallel = string(default="mpi")
 nodes = integer(default=4)
 # NOTE: always overrides value in [default]
 vars = string(default="")
+# Model component having generated input files (eam, eamxx, elm, mosart, ...)
+input_component = string(default="")
 
   [[__many__]]
   exclude = boolean(default=None)
   nodes = integer(default=None)
   vars = string(default=None)
+  input_component = string(default=None)
 
 [ts]
 area_nm = string(default="area")
@@ -93,6 +96,8 @@ extra_vars = string(default="")
 # Time-steps per day
 tpd = integer(default=1)
 ts_fmt = string(default="ts_only")
+# Model component having generated input files (eam, eamxx, elm, mosart, ...)
+input_component = string(default="")
 
   [[__many__]]
   area_nm = string(default=None)
@@ -101,6 +106,7 @@ ts_fmt = string(default="ts_only")
   extra_vars = string(default=None)
   tpd = integer(default=None)
   ts_fmt = string(default=None)
+  input_component = string(default=None)
 
 [tc_analysis]
 # NOTE: always overrides value in [default]

diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash
@@ -110,11 +110,7 @@ cat input.txt | ncclimo \
 --dpf={{ dpf }} \
 --tpd={{ tpd }} \
 {%- endif %}
-{%- if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' or input_files.split(".")[0] == 'elm' or input_files.split(".")[0] == 'clm2'  %}
---prc_typ={{ input_files.split(".")[0][:3] }}
-{%- else %}
---prc_typ=sgs
-{%- endif %}
+--prc_typ={{ prc_typ }}
 
 
 

diff --git a/zppy/ts.py b/zppy/ts.py
@@ -57,8 +57,10 @@ def ts(config, scriptDir, existing_bundles, job_ids_file):
                         % (c["mapping_file"])
                     )
 
-        # Component
-        c["component"] = getComponent(c["input_files"])
+        # Output component (for directory structure) and procedure type for ncclimo
+        c["component"], c["prc_typ"] = getComponent(
+            c["input_component"], c["input_files"]
+        )
 
         c["cmor_tables_prefix"] = c["diagnostics_base_path"]
 

diff --git a/zppy/utils.py b/zppy/utils.py
@@ -136,26 +136,37 @@ def getYears(years_list):
 
 
 # -----------------------------------------------------------------------------
-# Return component name from input files (e.g. 'cam.h0', 'clm2.h0', ...)
+# Return output component name and procedure type based on either
+# input_component or input_files
 
 
-def getComponent(input_files):
+def getComponent(input_component, input_files):
 
-    tmp = input_files.split(".")[0]
-    if tmp in ("cam", "eam"):
+    if input_component != "":
+        tmp = input_component
+    else:
+        tmp = input_files.split(".")[0]
+
+    # Default ncclim procedure type is "sgs"
+    prc_typ = "sgs"
+
+    # Output component (for directory structure) and ncclimo procedure type
+    if tmp in ("cam", "eam", "eamxx"):
         component = "atm"
+        prc_typ = tmp
     elif tmp in ("cpl",):
         component = "cpl"
     elif tmp in ("clm2", "elm"):
         component = "lnd"
+        prc_typ = tmp
     elif tmp in ("mosart",):
         component = "rof"
     else:
         raise ValueError(
-            "Cannot extract component name from input_files %s" % (input_files)
+            f"Cannot extract output component name from {input_component} or {input_files}."
         )
 
-    return component
+    return component, prc_typ
 
 
 # -----------------------------------------------------------------------------