Add e3sm_to_cmip task

tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg

@@ -78,24 +78,18 @@ years = "1985:1995:5"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 walltime = "00:50:00"
-# ts is in 5 year increments
+ts_num_years = 5
 years = "1985:1995:5"
 
   [[ atm_monthly_180x360_aave ]]
-  # for e3sm_diags (depend_on_ts), ilamb
-  frequency = "monthly"
+  # for ilamb
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
 
   [[ land_monthly ]]
   # for ilamb
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [e3sm_diags]
 active = True

tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg

@@ -101,19 +101,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/integration/generated/test_min_case_ilamb_chrysalis.cfg

@@ -32,21 +32,16 @@ years = "1985:2014:30"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 30
 walltime = "00:50:00"
 years = "1985:2014:30"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [ilamb]
 active = True

tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg

@@ -26,15 +26,13 @@ years = "1985:2014:30"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 30
 walltime = "00:50:00"
 years = "1985:2014:30"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [ilamb]
 active = True

tests/integration/generated/test_weekly_bundles_chrysalis.cfg

@@ -96,19 +96,15 @@ years = "1985:1989:2",
 [e3sm_to_cmip]
 active = True
 bundle = "bundle1"
+frequency = "monthly"
+ts_num_years = 2
 years = "1985:1989:2",
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "LAISHA,LAISUN"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg

@@ -83,19 +83,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  vars = "LAISHA,LAISUN"
 
 [tc_analysis]
 active = True

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -87,20 +87,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/integration/template_min_case_add_dependencies.cfg

@@ -78,24 +78,18 @@ years = "1985:1995:5"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 walltime = "00:50:00"
-# ts is in 5 year increments
+ts_num_years = 5
 years = "1985:1995:5"
 
   [[ atm_monthly_180x360_aave ]]
-  # for e3sm_diags (depend_on_ts), ilamb
-  frequency = "monthly"
+  # for ilamb
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
 
   [[ land_monthly ]]
   # for ilamb
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [e3sm_diags]
 active = True

tests/integration/template_min_case_carryover_dependencies.cfg

@@ -101,19 +101,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/integration/template_min_case_ilamb.cfg

@@ -32,21 +32,16 @@ years = "1985:2014:30"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 30
 walltime = "00:50:00"
 years = "1985:2014:30"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [ilamb]
 active = True

tests/integration/template_min_case_ilamb_land_only.cfg

@@ -26,15 +26,13 @@ years = "1985:2014:30"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
+ts_num_years = 30
 walltime = "00:50:00"
 years = "1985:2014:30"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [ilamb]
 active = True

tests/integration/template_weekly_bundles.cfg

@@ -96,19 +96,15 @@ years = "1985:1989:2",
 [e3sm_to_cmip]
 active = True
 bundle = "bundle1"
+frequency = "monthly"
+ts_num_years = 2
 years = "1985:1989:2",
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "LAISHA,LAISUN"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/integration/template_weekly_comprehensive_v2.cfg

@@ -83,19 +83,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  vars = "LAISHA,LAISUN"
 
 [tc_analysis]
 active = True

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -87,20 +87,15 @@ walltime = "00:30:00"
 
 [e3sm_to_cmip]
 active = True
+frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
 
   [[ land_monthly ]]
-  frequency = "monthly"
   input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 # TODO: Add "tc_analysis" back in after empty dat is resolved.
 # [tc_analysis]

tests/test_zppy_e3sm_diags.py

@@ -548,7 +548,6 @@ def test_add_ts_dependencies():
         add_ts_dependencies(c, dependencies, "script_dir", 1980)
         assert dependencies == [
             "script_dir/ts_sub_1980-1984-0005.status",
-            "script_dir/e3sm_to_cmip_sub_1980-1984-0005.status",
         ]
 
     c = {"sets": ["streamflow"]}

zppy/defaults/default.ini

@@ -119,6 +119,8 @@ input_component = string(default="")
 [e3sm_to_cmip]
 cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json")
 # Model component having generated input files (eam, eamxx, elm, mosart, ...)
+cmip_var_atm = string(default="pr, tas, rsds, rlds, rsus")
+cmip_var_land = string(default="mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh")
 input_component = string(default="")
 
   [[__many__]]

zppy/e3sm_diags.py

@@ -295,16 +295,6 @@ def add_ts_dependencies(
             end_yr,
             c["ts_num_years"],
         )
-        # e3sm_to_cmip task
-        add_dependencies(
-            dependencies,
-            script_dir,
-            "e3sm_to_cmip",
-            ts_sub,  # Relies on having the same subsection name as the corresponding ts subsection!
-            start_yr,
-            end_yr,
-            c["ts_num_years"],
-        )
     if "streamflow" in c["sets"]:
         add_dependencies(
             dependencies,

zppy/e3sm_to_cmip.py

@@ -13,9 +13,6 @@
     get_years,
     initialize_template,
     make_executable,
-    set_component_and_prc_typ,
-    set_grid,
-    set_mapping_file,
     submit_script,
     write_settings_file,
 )
@@ -34,9 +31,6 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f
     # --- Generate and submit e3sm_to_cmip scripts ---
     for c in tasks:
         dependencies: List[str] = []
-        set_mapping_file(c)
-        set_grid(c)
-        set_component_and_prc_typ(c)
         c["cmor_tables_prefix"] = c["diagnostics_base_path"]
         year_sets: List[Tuple[int, int]] = get_years(c["years"])
         # Loop over year sets

zppy/templates/e3sm_to_cmip.bash

@@ -33,13 +33,13 @@ EOF
   ${dest_cmip}/${tmp_dir} \
   {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%}
   --var-list \
-  'mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \
+  '{{ cmip_var_lnd }}' \
   --realm \
   lnd \
   {% endif -%}
   {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%}
   --var-list \
-  'pr, tas, rsds, rlds, rsus' \
+  '{{ cmip_var_atm }}' \
   --realm \
   atm \
   {% endif -%}

zppy/ts.py

@@ -35,7 +35,6 @@ def ts(config: ConfigObj, script_dir: str, existing_bundles, job_ids_file):
         set_mapping_file(c)
         set_grid(c)
         set_component_and_prc_typ(c)
-        c["cmor_tables_prefix"] = c["diagnostics_base_path"]
         year_sets: List[Tuple[int, int]] = get_years(c["years"])
         # Loop over year sets
         for s in year_sets: