Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add e3sm_to_cmip task #650

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Add e3sm_to_cmip task #650

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e99178b
Add e3sm_to_cmip task
forsyth2 Dec 7, 2024
7fc7e2b
Add dependency handling
forsyth2 Dec 10, 2024
3a8dc06
Update defaults
forsyth2 Dec 10, 2024
e7d43e1
Fix cmortables_dir
forsyth2 Dec 10, 2024
f92ad22
Currently testing on v3
forsyth2 Dec 11, 2024
2cbc136
Image check tests passing
forsyth2 Dec 12, 2024
e93a577
Clean up code
forsyth2 Dec 12, 2024
5812010
Address comments
forsyth2 Dec 12, 2024
44c93e0
Update parameters
forsyth2 Dec 12, 2024
6e37e19
Handle ts dependency and output
forsyth2 Dec 13, 2024
c1116e9
Unit tests passing
forsyth2 Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,24 +78,18 @@ years = "1985:1995:5"

[e3sm_to_cmip]
active = True
frequency = "monthly"
walltime = "00:50:00"
# ts is in 5 year increments
ts_num_years = 5
years = "1985:1995:5"

[[ atm_monthly_180x360_aave ]]
# for e3sm_diags (depend_on_ts), ilamb
frequency = "monthly"
# for ilamb
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
# for ilamb
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_diags]
active = True
Expand Down
8 changes: 2 additions & 6 deletions tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
9 changes: 2 additions & 7 deletions tests/integration/generated/test_min_case_ilamb_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,16 @@ years = "1985:2014:30"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 30
walltime = "00:50:00"
years = "1985:2014:30"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
Expand Down
6 changes: 2 additions & 4 deletions tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@ years = "1985:2014:30"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 30
walltime = "00:50:00"
years = "1985:2014:30"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
Expand Down
8 changes: 2 additions & 6 deletions tests/integration/generated/test_weekly_bundles_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,15 @@ years = "1985:1989:2",
[e3sm_to_cmip]
active = True
bundle = "bundle1"
frequency = "monthly"
ts_num_years = 2
years = "1985:1989:2",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "LAISHA,LAISUN"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
6 changes: 1 addition & 5 deletions tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,19 +83,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years=2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
vars = "LAISHA,LAISUN"

[tc_analysis]
active = True
Expand Down
7 changes: 1 addition & 6 deletions tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years=2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
12 changes: 3 additions & 9 deletions tests/integration/template_min_case_add_dependencies.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,24 +78,18 @@ years = "1985:1995:5"

[e3sm_to_cmip]
active = True
frequency = "monthly"
walltime = "00:50:00"
# ts is in 5 year increments
ts_num_years = 5
years = "1985:1995:5"

[[ atm_monthly_180x360_aave ]]
# for e3sm_diags (depend_on_ts), ilamb
frequency = "monthly"
# for ilamb
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
# for ilamb
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_diags]
active = True
Expand Down
8 changes: 2 additions & 6 deletions tests/integration/template_min_case_carryover_dependencies.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
9 changes: 2 additions & 7 deletions tests/integration/template_min_case_ilamb.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,16 @@ years = "1985:2014:30"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 30
walltime = "00:50:00"
years = "1985:2014:30"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
Expand Down
6 changes: 2 additions & 4 deletions tests/integration/template_min_case_ilamb_land_only.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@ years = "1985:2014:30"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years = 30
walltime = "00:50:00"
years = "1985:2014:30"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
Expand Down
8 changes: 2 additions & 6 deletions tests/integration/template_weekly_bundles.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,15 @@ years = "1985:1989:2",
[e3sm_to_cmip]
active = True
bundle = "bundle1"
frequency = "monthly"
ts_num_years = 2
years = "1985:1989:2",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "LAISHA,LAISUN"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
6 changes: 1 addition & 5 deletions tests/integration/template_weekly_comprehensive_v2.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,19 +83,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years=2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
vars = "LAISHA,LAISUN"

[tc_analysis]
active = True
Expand Down
7 changes: 1 addition & 6 deletions tests/integration/template_weekly_comprehensive_v3.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,15 @@ walltime = "00:30:00"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years=2
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
Copy link
Collaborator

@chengzhuzhang chengzhuzhang Dec 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@forsyth2 The updates look good. Thank you. I'm curious if input_files is required in this sub-session?

Copy link
Collaborator Author

@forsyth2 forsyth2 Dec 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's used in e3sm_to_cmip.bash:

  {% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%}
  --var-list \
  '{{ cmip_var_lnd }}' \
  --realm \
  lnd \
  {% endif -%}
  {% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%}
  --var-list \
  '{{ cmip_var_atm }}' \
  --realm \
  atm \
  {% endif -%}

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I see, it is used for determine realm and var-list accordingly. Thanks.

input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
Expand Down
1 change: 0 additions & 1 deletion tests/test_zppy_e3sm_diags.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,6 @@ def test_add_ts_dependencies():
add_ts_dependencies(c, dependencies, "script_dir", 1980)
assert dependencies == [
"script_dir/ts_sub_1980-1984-0005.status",
"script_dir/e3sm_to_cmip_sub_1980-1984-0005.status",
]

c = {"sets": ["streamflow"]}
Expand Down
2 changes: 2 additions & 0 deletions zppy/defaults/default.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ input_component = string(default="")
[e3sm_to_cmip]
cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json")
# Model component having generated input files (eam, eamxx, elm, mosart, ...)
cmip_var_atm = string(default="pr, tas, rsds, rlds, rsus")
cmip_var_land = string(default="mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh")
input_component = string(default="")

[[__many__]]
Expand Down
10 changes: 0 additions & 10 deletions zppy/e3sm_diags.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,16 +295,6 @@ def add_ts_dependencies(
end_yr,
c["ts_num_years"],
)
# e3sm_to_cmip task
add_dependencies(
dependencies,
script_dir,
"e3sm_to_cmip",
ts_sub, # Relies on having the same subsection name as the corresponding ts subsection!
start_yr,
end_yr,
c["ts_num_years"],
)
if "streamflow" in c["sets"]:
add_dependencies(
dependencies,
Expand Down
6 changes: 0 additions & 6 deletions zppy/e3sm_to_cmip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
get_years,
initialize_template,
make_executable,
set_component_and_prc_typ,
set_grid,
set_mapping_file,
submit_script,
write_settings_file,
)
Expand All @@ -34,9 +31,6 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f
# --- Generate and submit e3sm_to_cmip scripts ---
for c in tasks:
dependencies: List[str] = []
set_mapping_file(c)
set_grid(c)
set_component_and_prc_typ(c)
c["cmor_tables_prefix"] = c["diagnostics_base_path"]
year_sets: List[Tuple[int, int]] = get_years(c["years"])
# Loop over year sets
Expand Down
4 changes: 2 additions & 2 deletions zppy/templates/e3sm_to_cmip.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ EOF
${dest_cmip}/${tmp_dir} \
{% if input_files.split(".")[0] == 'clm2' or input_files.split(".")[0] == 'elm' -%}
--var-list \
'mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh' \
'{{ cmip_var_lnd }}' \
--realm \
lnd \
{% endif -%}
{% if input_files.split(".")[0] == 'cam' or input_files.split(".")[0] == 'eam' -%}
--var-list \
'pr, tas, rsds, rlds, rsus' \
'{{ cmip_var_atm }}' \
--realm \
atm \
{% endif -%}
Expand Down
1 change: 0 additions & 1 deletion zppy/ts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ def ts(config: ConfigObj, script_dir: str, existing_bundles, job_ids_file):
set_mapping_file(c)
set_grid(c)
set_component_and_prc_typ(c)
c["cmor_tables_prefix"] = c["diagnostics_base_path"]
year_sets: List[Tuple[int, int]] = get_years(c["years"])
# Loop over year sets
for s in year_sets:
Expand Down
Loading