diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index a3795e4c6b..c00226b7dd 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -7,3 +7,8 @@ e44dc469439e02e9ee582dab274d890ebdfab104
b88e1cd1b28e3609684c79a2ec0e88f26cfc362b
51c102c5df2e0ef971b5f8eeeb477567899af63a
7dacad70e74e2ec97f6492d4e7a3cb5dd498bcd7
+b771971e3299c4fa56534b93421f7a2b9c7282fd
+9de88bb57ea9855da408cbec1dc8acb9079eda47
+8bc4688e52ea23ef688e283698f70a44388373eb
+# Ran SystemTests and python/ctsm through black python formatter
+5364ad66eaceb55dde2d3d598fe4ce37ac83a93c
diff --git a/Externals.cfg b/Externals.cfg
index cd6855c9dc..62a119004e 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -44,7 +44,7 @@ required = True
local_path = cime
protocol = git
repo_url = https://github.com/ESMCI/cime
-tag = cime6.0.108
+tag = cime6.0.125
required = True
[cmeps]
diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm
index 5adcf70ede..256de592c6 100755
--- a/bld/CLMBuildNamelist.pm
+++ b/bld/CLMBuildNamelist.pm
@@ -1625,9 +1625,9 @@ sub process_namelist_inline_logic {
setup_logic_urban($opts, $nl_flags, $definition, $defaults, $nl);
###############################
- # namelist group: crop #
+ # namelist group: crop_inparm #
###############################
- setup_logic_crop($opts, $nl_flags, $definition, $defaults, $nl);
+ setup_logic_crop_inparm($opts, $nl_flags, $definition, $defaults, $nl);
###############################
# namelist group: ch4par_in #
@@ -1695,6 +1695,11 @@ sub process_namelist_inline_logic {
##################################
setup_logic_lai_streams($opts, $nl_flags, $definition, $defaults, $nl);
+ ##################################
+ # namelist group: cropcal_streams #
+ ##################################
+ setup_logic_cropcal_streams($opts, $nl_flags, $definition, $defaults, $nl);
+
##########################################
# namelist group: soil_moisture_streams #
##########################################
@@ -2149,7 +2154,7 @@ sub setup_logic_urban {
#-------------------------------------------------------------------------------
-sub setup_logic_crop {
+sub setup_logic_crop_inparm {
my ($opts, $nl_flags, $definition, $defaults, $nl) = @_;
if ( &value_is_true($nl->get_value('use_crop')) ) {
@@ -3920,6 +3925,53 @@ sub setup_logic_lai_streams {
#-------------------------------------------------------------------------------
+sub setup_logic_cropcal_streams {
+ my ($opts, $nl_flags, $definition, $defaults, $nl) = @_;
+
+ # Set first and last stream years
+ add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_first_cropcal',
+ 'sim_year'=>$nl_flags->{'sim_year'},
+ 'sim_year_range'=>$nl_flags->{'sim_year_range'});
+ add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_last_cropcal',
+ 'sim_year'=>$nl_flags->{'sim_year'},
+ 'sim_year_range'=>$nl_flags->{'sim_year_range'});
+
+ # Set align year, if first and last years are different
+ if ( $nl->get_value('stream_year_first_cropcal') !=
+ $nl->get_value('stream_year_last_cropcal') ) {
+ add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl,
+ 'model_year_align_cropcal', 'sim_year'=>$nl_flags->{'sim_year'},
+ 'sim_year_range'=>$nl_flags->{'sim_year_range'});
+ }
+
+ # Set up other crop calendar parameters
+ add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'generate_crop_gdds');
+ add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_mxmat');
+
+ # Option checks
+ my $generate_crop_gdds = $nl->get_value('generate_crop_gdds') ;
+ my $use_mxmat = $nl->get_value('use_mxmat') ;
+ my $sdate_file = $nl->get_value('stream_fldFileName_sdate') ;
+ my $gdd_file = $nl->get_value('stream_fldFileName_cultivar_gdds') ;
+ my $mesh_file = $nl->get_value('stream_meshfile_cropcal') ;
+ if ( $generate_crop_gdds eq '.true.' ) {
+ if ( $use_mxmat eq '.true.' ) {
+ $log->fatal_error("If generate_crop_gdds is true, you must also set use_mxmat to false" );
+ }
+ if ( $sdate_file eq '' ) {
+ $log->fatal_error("If generate_crop_gdds is true, you must specify stream_fldFileName_sdate")
+ }
+ if ( $gdd_file ne '' ) {
+ $log->fatal_error("If generate_crop_gdds is true, do not specify stream_fldFileName_cultivar_gdds")
+ }
+ }
+ if ( $mesh_file eq '' and ( $sdate_file ne '' or $gdd_file ne '' ) ) {
+ $log->fatal_error("If prescribing crop sowing dates and/or maturity requirements, you must specify stream_meshfile_cropcal")
+ }
+}
+
+#-------------------------------------------------------------------------------
+
sub setup_logic_soilwater_movement {
my ($opts, $nl_flags, $definition, $defaults, $nl) = @_;
@@ -4297,6 +4349,7 @@ sub write_output_files {
my @groups;
@groups = qw(clm_inparm ndepdyn_nml popd_streams urbantv_streams light_streams
soil_moisture_streams lai_streams atm2lnd_inparm lnd2atm_inparm clm_canopyhydrology_inparm cnphenology
+ cropcal_streams
clm_soilhydrology_inparm dynamic_subgrid cnvegcarbonstate
finidat_consistency_checks dynpft_consistency_checks
clm_initinterp_inparm century_soilbgcdecompcascade
@@ -4304,7 +4357,7 @@ sub write_output_files {
soilwater_movement_inparm rooting_profile_inparm
soil_resis_inparm bgc_shared canopyfluxes_inparm aerosol
clmu_inparm clm_soilstate_inparm clm_nitrogen clm_snowhydrology_inparm
- cnprecision_inparm clm_glacier_behavior crop irrigation_inparm
+ cnprecision_inparm clm_glacier_behavior crop_inparm irrigation_inparm
surfacealbedo_inparm water_tracers_inparm);
#@groups = qw(clm_inparm clm_canopyhydrology_inparm clm_soilhydrology_inparm
diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml
index f1d251e02b..3cf9a3ebc0 100644
--- a/bld/namelist_files/namelist_defaults_ctsm.xml
+++ b/bld/namelist_files/namelist_defaults_ctsm.xml
@@ -562,6 +562,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
DependsOnLat
.false.
Constant
+.false.
+.true.
.false.
@@ -1665,6 +1667,11 @@ use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1
nn
nn
+
+1850
+2100
+1850
+
none
diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml
index 3b56941113..dc693b50ba 100644
--- a/bld/namelist_files/namelist_definition_ctsm.xml
+++ b/bld/namelist_files/namelist_definition_ctsm.xml
@@ -1090,20 +1090,20 @@ Toggle to turn on the 1-year grain product pool in the crop model
+ group="crop_inparm" valid_values="constant,varytropicsbylat" value="constant">
Type of mapping to use for base temperature for prognostic crop model
constant = Just use baset from the PFT parameter file
varytropicsbylat = Vary the tropics by latitude
+ group="crop_inparm" valid_values="" value="0.4d00">
Only used when baset_mapping == varytropicsbylat
Slope with latitude in degrees to vary tropical baset by
+ group="crop_inparm" valid_values="" value="12.0d00">
Only used when baset_mapping == varytropicsbylat
Intercept at zero latitude to add to baset from the PFT parameter file
@@ -1129,6 +1129,16 @@ Phenology onset depends on the vegetation type
(only used when CN is on)
+
+Set to .true. in order to override crop harvesting logic and to instead harvest the day before the next sowing date. Used to generate growing-degree day outputs that can be used with an external script to generate new GDD requirement ("cultivar") files.
+
+
+
+Set to .false. in order to ignore crop PFT parameter for maximum growing season length (mxmat). Must be set to .false. when generate_crop_gdds is .true.
+
+
Method for determining what the minimum critical day length for seasonal decidious leaf offset depends on
@@ -1805,6 +1815,41 @@ Mapping method from LAI input file to the model resolution
copy = copy using the same indices
+
+
+
+
+
+
+First year to loop over for crop calendar data
+
+
+
+Last year to loop over for crop calendar data
+
+
+
+Simulation year that aligns with stream_year_first_cropcal value
+
+
+
+Filename of input stream data for sowing dates
+
+
+
+Filename of input stream data for cultivar growing degree-day targets
+
+
+
+Filename of input stream data for crop calendar inputs
+
+
diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py
index 70dfa7c434..d2a9c04312 100644
--- a/cime_config/SystemTests/fsurdatmodifyctsm.py
+++ b/cime_config/SystemTests/fsurdatmodifyctsm.py
@@ -5,7 +5,7 @@
import os
import re
-import subprocess
+import systemtest_utils as stu
from CIME.SystemTests.system_tests_common import SystemTestsCommon
from CIME.XML.standard_module_setup import *
from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files
@@ -69,33 +69,13 @@ def _run_modify_fsurdat(self):
tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier")
self._case.load_env(reset=True)
- conda_env = ". " + self._get_caseroot() + "/.env_mach_specific.sh; "
- # Preprend the commands to get the conda environment for python first
- conda_env += self._get_conda_env()
- # Source the env
- try:
- subprocess.run(
- conda_env + "python3 " + tool_path + " " + self._cfg_file_path,
- shell=True,
- check=True,
- )
- except subprocess.CalledProcessError as error:
- print("ERROR while getting the conda environment and/or ")
- print("running the fsurdat_modifier tool: ")
- print("(1) If your ctsm_pylib environment is out of date or you ")
- print("have not created the ctsm_pylib environment, yet, you may ")
- print("get past this error by running ./py_env_create ")
- print("in your ctsm directory and trying this test again. ")
- print("(2) If conda is not available, install and load conda, ")
- print("run ./py_env_create, and then try this test again. ")
- print("(3) If (1) and (2) are not the issue, then you may be ")
- print("getting an error within the fsurdat_modifier tool itself. ")
- print("Default error message: ")
- print(error.output)
- raise
- except:
- print("ERROR trying to run fsurdat_modifier tool.")
- raise
+ command = f"python3 {tool_path} {self._cfg_file_path}"
+ stu.run_python_script(
+ self._get_caseroot(),
+ "ctsm_pylib",
+ command,
+ tool_path,
+ )
def _modify_user_nl(self):
append_to_user_nl_files(
@@ -103,24 +83,3 @@ def _modify_user_nl(self):
component="clm",
contents="fsurdat = '{}'".format(self._fsurdat_out),
)
-
- def _get_conda_env(self):
- #
- # Add specific commands needed on different machines to get conda available
- # Use semicolon here since it's OK to fail
- #
- # Execute the module unload/load when "which conda" fails
- # eg on cheyenne
- try:
- subprocess.run("which conda", shell=True, check=True)
- conda_env = " "
- except subprocess.CalledProcessError:
- # Remove python and add conda to environment for cheyennne
- conda_env = "module unload python; module load conda;"
-
- # Activate the python environment
- conda_env += " conda activate ctsm_pylib"
- # End above to get to actual command
- conda_env += " && "
-
- return conda_env
diff --git a/cime_config/SystemTests/lreprstruct.py b/cime_config/SystemTests/lreprstruct.py
index f8a3300e56..a03fb1815b 100644
--- a/cime_config/SystemTests/lreprstruct.py
+++ b/cime_config/SystemTests/lreprstruct.py
@@ -49,6 +49,20 @@ def _case_one_setup(self):
contents="for_testing_use_repr_structure_pool=.true.",
)
+ # Replace any GRAIN outputs with the same outputs for REPRODUCTIVE1 and REPRODUCTIVE2
+ user_nl_clm_path = os.path.join(self._get_caseroot(), "user_nl_clm")
+ with open(user_nl_clm_path) as f:
+ user_nl_clm_text = f.read()
+ for grain_output in re.findall("GRAIN\w*", user_nl_clm_text):
+ user_nl_clm_text = user_nl_clm_text.replace(
+ grain_output,
+ grain_output.replace("GRAIN", "REPRODUCTIVE1")
+ + "', '"
+ + grain_output.replace("GRAIN", "REPRODUCTIVE2"),
+ )
+ with open(user_nl_clm_path, "w") as f:
+ f.write(user_nl_clm_text)
+
def _case_two_setup(self):
# This is needed in the nearly-standard case to prevent grain from being used to
# replenish crop seed deficits, thus making grain act like the reproductive
diff --git a/cime_config/SystemTests/rxcropmaturity.py b/cime_config/SystemTests/rxcropmaturity.py
new file mode 100644
index 0000000000..4fd812b84a
--- /dev/null
+++ b/cime_config/SystemTests/rxcropmaturity.py
@@ -0,0 +1,399 @@
+"""
+CTSM-specific test that first performs a GDD-generating run, then calls
+Python code to generate the maturity requirement file. This is then used
+in a sowing+maturity forced run, which finally is tested to ensure
+correct behavior.
+
+Currently only supports 0.9x1.25, 1.9x2.5, and 10x15 resolutions. Eventually,
+this test should be able to generate its own files at whatever resolution it's
+called at. Well, really, the ultimate goal would be to give CLM the files
+at the original resolution (for GGCMI phase 3, 0.5°) and have the stream
+code do the interpolation. However, that wouldn't act on harvest dates
+(which are needed for generate_gdds.py). I could have Python interpolate
+those, but this would cause a potential inconsistency.
+"""
+
+import os
+import re
+import systemtest_utils as stu
+import subprocess
+from CIME.SystemTests.system_tests_common import SystemTestsCommon
+from CIME.XML.standard_module_setup import *
+from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files
+import shutil, glob
+
+logger = logging.getLogger(__name__)
+
+
+class RXCROPMATURITY(SystemTestsCommon):
+ def __init__(self, case):
+ # initialize an object interface to the SMS system test
+ SystemTestsCommon.__init__(self, case)
+
+ # Ensure run length is at least 5 years. Minimum to produce one complete growing season (i.e., two complete calendar years) actually 4 years, but that only gets you 1 season usable for GDD generation, so you can't check for season-to-season consistency.
+ stop_n = self._case.get_value("STOP_N")
+ stop_option = self._case.get_value("STOP_OPTION")
+ stop_n_orig = stop_n
+ stop_option_orig = stop_option
+ if "nsecond" in stop_option:
+ stop_n /= 60
+ stop_option = "nminutes"
+ if "nminute" in stop_option:
+ stop_n /= 60
+ stop_option = "nhours"
+ if "nhour" in stop_option:
+ stop_n /= 24
+ stop_option = "ndays"
+ if "nday" in stop_option:
+ stop_n /= 365
+ stop_option = "nyears"
+ if "nmonth" in stop_option:
+ stop_n /= 12
+ stop_option = "nyears"
+ error_message = None
+ if "nyear" not in stop_option:
+ error_message = (
+ f"STOP_OPTION ({stop_option_orig}) must be nsecond(s), nminute(s), "
+ + "nhour(s), nday(s), nmonth(s), or nyear(s)"
+ )
+ elif stop_n < 5:
+ error_message = (
+ "RXCROPMATURITY must be run for at least 5 years; you requested "
+ + f"{stop_n_orig} {stop_option_orig[1:]}"
+ )
+ if error_message is not None:
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ # Get the number of complete years that will be run
+ self._run_Nyears = int(stop_n)
+
+ # Only allow RXCROPMATURITY to be called with test cropMonthOutput
+ casebaseid = self._case.get_value("CASEBASEID")
+ if casebaseid.split("-")[-1] != "cropMonthOutput":
+ error_message = (
+ "Only call RXCROPMATURITY with test cropMonthOutput "
+ + "to avoid potentially huge sets of daily outputs."
+ )
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ # Get files with prescribed sowing and harvest dates
+ self._get_rx_dates()
+
+ # Which conda environment should we use?
+ self._get_conda_env()
+
+ def run_phase(self):
+ # Modeling this after the SSP test, we create a clone to be the case whose outputs we don't
+ # want to be saved as baseline.
+
+ # -------------------------------------------------------------------
+ # (1) Set up GDD-generating run
+ # -------------------------------------------------------------------
+ # Create clone to be GDD-Generating case
+ logger.info("RXCROPMATURITY log: cloning setup")
+ case_rxboth = self._case
+ caseroot = self._case.get_value("CASEROOT")
+ clone_path = f"{caseroot}.gddgen"
+ self._path_gddgen = clone_path
+ if os.path.exists(self._path_gddgen):
+ shutil.rmtree(self._path_gddgen)
+ logger.info("RXCROPMATURITY log: cloning")
+ case_gddgen = self._case.create_clone(clone_path, keepexe=True)
+ logger.info("RXCROPMATURITY log: done cloning")
+
+ os.chdir(self._path_gddgen)
+ self._set_active_case(case_gddgen)
+
+ # Set up stuff that applies to both tests
+ self._setup_all()
+
+ # Add stuff specific to GDD-Generating run
+ logger.info("RXCROPMATURITY log: modify user_nl files: generate GDDs")
+ self._append_to_user_nl_clm(
+ [
+ "generate_crop_gdds = .true.",
+ "use_mxmat = .false.",
+ " ",
+ "! (h2) Daily outputs for GDD generation and figure-making",
+ "hist_fincl3 = 'GDDACCUM', 'GDDHARV'",
+ "hist_nhtfrq(3) = -24",
+ "hist_mfilt(3) = 365",
+ "hist_type1d_pertape(3) = 'PFTS'",
+ "hist_dov2xy(3) = .false.",
+ ]
+ )
+
+ # If flanduse_timeseries is defined, we need to make a static version for this test. This
+ # should have every crop in most of the world.
+ self._get_flanduse_timeseries_in(case_gddgen)
+ if self._flanduse_timeseries_in is not None:
+
+ # Download files from the server, if needed
+ case_gddgen.check_all_input_data()
+
+ # Make custom version of surface file
+ logger.info("RXCROPMATURITY log: run make_fsurdat_all_crops_everywhere")
+ self._run_make_fsurdat_all_crops_everywhere()
+
+ # -------------------------------------------------------------------
+ # (2) Perform GDD-generating run and generate prescribed GDDs file
+ # -------------------------------------------------------------------
+ logger.info("RXCROPMATURITY log: Start GDD-Generating run")
+
+ # As per SSP test:
+ # "No history files expected, set suffix=None to avoid compare error"
+ # We *do* expect history files here, but anyway. This works.
+ self._skip_pnl = False
+ self.run_indv(suffix=None, st_archive=True)
+
+ self._run_generate_gdds(case_gddgen)
+
+ # -------------------------------------------------------------------
+ # (3) Set up and perform Prescribed Calendars run
+ # -------------------------------------------------------------------
+ os.chdir(caseroot)
+ self._set_active_case(case_rxboth)
+
+ # Set up stuff that applies to both tests
+ self._setup_all()
+
+ # Add stuff specific to Prescribed Calendars run
+ logger.info("RXCROPMATURITY log: modify user_nl files: Prescribed Calendars")
+ self._append_to_user_nl_clm(
+ [
+ "generate_crop_gdds = .false.",
+ f"stream_fldFileName_cultivar_gdds = '{self._gdds_file}'",
+ ]
+ )
+
+ self.run_indv()
+
+ # -------------------------------------------------------------------
+ # (4) Check Prescribed Calendars run
+ # -------------------------------------------------------------------
+ logger.info("RXCROPMATURITY log: output check: Prescribed Calendars")
+ self._run_check_rxboth_run()
+
+ # Get sowing and harvest dates for this resolution.
+ def _get_rx_dates(self):
+ # Eventually, I want to remove these hard-coded resolutions so that this test can generate
+ # its own sowing and harvest date files at whatever resolution is requested.
+ lnd_grid = self._case.get_value("LND_GRID")
+ input_data_root = self._case.get_value("DIN_LOC_ROOT")
+ processed_crop_dates_dir = f"{input_data_root}/lnd/clm2/cropdata/calendars/processed"
+ if lnd_grid == "10x15":
+ self._sdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "sdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f10_f10_mg37.2000-2000.20230330_165301.nc",
+ )
+ self._hdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "hdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f10_f10_mg37.2000-2000.20230330_165301.nc",
+ )
+ elif lnd_grid == "1.9x2.5":
+ self._sdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "sdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f19_g17.2000-2000.20230102_175625.nc",
+ )
+ self._hdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "hdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f19_g17.2000-2000.20230102_175625.nc",
+ )
+ elif lnd_grid == "0.9x1.25":
+ self._sdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "sdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f09_g17.2000-2000.20230520_134417.nc",
+ )
+ self._hdatefile = os.path.join(
+ processed_crop_dates_dir,
+ "hdates_ggcmi_crop_calendar_phase3_v1.01_nninterp-f09_g17.2000-2000.20230520_134418.nc",
+ )
+ else:
+ error_message = "ERROR: RXCROPMATURITY currently only supports 0.9x1.25, 1.9x2.5, and 10x15 resolutions"
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ # Ensure files exist
+ error_message = None
+ if not os.path.exists(self._sdatefile):
+ error_message = f"ERROR: Sowing date file not found: {self._sdatefile}"
+ elif not os.path.exists(self._hdatefile):
+ error_message = f"ERROR: Harvest date file not found: {self._sdatefile}"
+ if error_message is not None:
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ def _setup_all(self):
+ logger.info("RXCROPMATURITY log: _setup_all start")
+
+ # Get some info
+ self._ctsm_root = self._case.get_value("COMP_ROOT_DIR_LND")
+ run_startdate = self._case.get_value("RUN_STARTDATE")
+ self._run_startyear = int(run_startdate.split("-")[0])
+
+ # Set sowing dates file (and other crop calendar settings) for all runs
+ logger.info("RXCROPMATURITY log: modify user_nl files: all tests")
+ self._modify_user_nl_allruns()
+ logger.info("RXCROPMATURITY log: _setup_all done")
+
+ # Make a surface dataset that has every crop in every gridcell
+ def _run_make_fsurdat_all_crops_everywhere(self):
+
+ # fsurdat should be defined. Where is it?
+ self._fsurdat_in = None
+ with open(self._lnd_in_path, "r") as lnd_in:
+ for line in lnd_in:
+ fsurdat_in = re.match(r" *fsurdat *= *'(.*)'", line)
+ if fsurdat_in:
+ self._fsurdat_in = fsurdat_in.group(1)
+ break
+ if self._fsurdat_in is None:
+ error_message = "fsurdat not defined"
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ # Where we will save the fsurdat version for this test
+ self._fsurdat_out = os.path.join(self._path_gddgen, "fsurdat.nc")
+
+ # Make fsurdat for this test, if not already done
+ if not os.path.exists(self._fsurdat_out):
+ tool_path = os.path.join(
+ self._ctsm_root,
+ "python",
+ "ctsm",
+ "crop_calendars",
+ "make_fsurdat_all_crops_everywhere.py",
+ )
+ command = (
+ f"python3 {tool_path} " + f"-i {self._fsurdat_in} " + f"-o {self._fsurdat_out}"
+ )
+ stu.run_python_script(
+ self._get_caseroot(),
+ self._this_conda_env,
+ command,
+ tool_path,
+ )
+
+ # Modify namelist
+ logger.info("RXCROPMATURITY log: modify user_nl files: new fsurdat")
+ self._append_to_user_nl_clm(
+ [
+ "fsurdat = '{}'".format(self._fsurdat_out),
+ "do_transient_crops = .false.",
+ "flanduse_timeseries = ''",
+ "use_init_interp = .true.",
+ ]
+ )
+
+ def _run_check_rxboth_run(self):
+
+ output_dir = os.path.join(self._get_caseroot(), "run")
+ first_usable_year = self._run_startyear + 2
+ last_usable_year = self._run_startyear + self._run_Nyears - 2
+
+ tool_path = os.path.join(
+ self._ctsm_root, "python", "ctsm", "crop_calendars", "check_rxboth_run.py"
+ )
+ command = (
+ f"python3 {tool_path} "
+ + f"--directory {output_dir} "
+ + f"-y1 {first_usable_year} "
+ + f"-yN {last_usable_year} "
+ + f"--rx-sdates-file {self._sdatefile} "
+ + f"--rx-gdds-file {self._gdds_file} "
+ )
+ stu.run_python_script(
+ self._get_caseroot(),
+ self._this_conda_env,
+ command,
+ tool_path,
+ )
+
+ def _modify_user_nl_allruns(self):
+ nl_additions = [
+ "stream_meshfile_cropcal = '{}'".format(self._case.get_value("LND_DOMAIN_MESH")),
+ "stream_fldFileName_sdate = '{}'".format(self._sdatefile),
+ "stream_year_first_cropcal = 2000",
+ "stream_year_last_cropcal = 2000",
+ "model_year_align_cropcal = 2000",
+ " ",
+ "! (h1) Annual outputs on sowing or harvest axis",
+ "hist_fincl2 = 'GRAINC_TO_FOOD_PERHARV', 'GRAINC_TO_FOOD_ANN', 'SDATES', 'SDATES_PERHARV', 'SYEARS_PERHARV', 'HDATES', 'GDDHARV_PERHARV', 'GDDACCUM_PERHARV', 'HUI_PERHARV', 'SOWING_REASON_PERHARV', 'HARVEST_REASON_PERHARV'",
+ "hist_nhtfrq(2) = 17520",
+ "hist_mfilt(2) = 999",
+ "hist_type1d_pertape(2) = 'PFTS'",
+ "hist_dov2xy(2) = .false.",
+ ]
+ self._append_to_user_nl_clm(nl_additions)
+
+ def _run_generate_gdds(self, case_gddgen):
+ self._generate_gdds_dir = os.path.join(self._path_gddgen, "generate_gdds_out")
+ os.makedirs(self._generate_gdds_dir)
+
+ # Get arguments to generate_gdds.py
+ dout_sr = case_gddgen.get_value("DOUT_S_ROOT")
+ input_dir = os.path.join(dout_sr, "lnd", "hist")
+ first_season = self._run_startyear + 2
+ last_season = self._run_startyear + self._run_Nyears - 2
+ sdates_file = self._sdatefile
+ hdates_file = self._hdatefile
+
+ # It'd be much nicer to call generate_gdds.main(), but I can't import generate_gdds.
+ tool_path = os.path.join(
+ self._ctsm_root, "python", "ctsm", "crop_calendars", "generate_gdds.py"
+ )
+ command = " ".join(
+ [
+ f"python3 {tool_path}",
+ f"--input-dir {input_dir}",
+ f"--first-season {first_season}",
+ f"--last-season {last_season}",
+ f"--sdates-file {sdates_file}",
+ f"--hdates-file {hdates_file}",
+ f"--output-dir generate_gdds_out",
+ f"--skip-crops miscanthus,irrigated_miscanthus"
+ ]
+ )
+ stu.run_python_script(
+ self._get_caseroot(),
+ self._this_conda_env,
+ command,
+ tool_path,
+ )
+
+ # Where were the prescribed maturity requirements saved?
+ generated_gdd_files = glob.glob(os.path.join(self._generate_gdds_dir, "gdds_*.nc"))
+ if len(generated_gdd_files) != 1:
+ error_message = f"ERROR: Expected one matching prescribed maturity requirements file; found {len(generated_gdd_files)}: {generated_gdd_files}"
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+ self._gdds_file = generated_gdd_files[0]
+
+ def _get_conda_env(self):
+ conda_setup_commands = stu.cmds_to_setup_conda(self._get_caseroot())
+
+ # If npl conda environment is available, use that (It has dask, which
+ # enables chunking, which makes reading daily 1-degree netCDF files
+ # much more efficient.
+ if "npl " in os.popen(conda_setup_commands + "conda env list").read():
+ self._this_conda_env = "npl"
+ else:
+ self._this_conda_env = "ctsm_pylib"
+
+ def _append_to_user_nl_clm(self, additions):
+ caseroot = self._get_caseroot()
+ append_to_user_nl_files(caseroot=caseroot, component="clm", contents=additions)
+
+ # Is flanduse_timeseries defined? If so, where is it?
+ def _get_flanduse_timeseries_in(self, case):
+ case.create_namelists(component="lnd")
+ self._lnd_in_path = os.path.join(self._path_gddgen, "CaseDocs", "lnd_in")
+ self._flanduse_timeseries_in = None
+ with open(self._lnd_in_path, "r") as lnd_in:
+ for line in lnd_in:
+ flanduse_timeseries_in = re.match(r" *flanduse_timeseries *= *'(.*)'", line)
+ if flanduse_timeseries_in:
+ self._flanduse_timeseries_in = flanduse_timeseries_in.group(1)
+ break
diff --git a/cime_config/SystemTests/systemtest_utils.py b/cime_config/SystemTests/systemtest_utils.py
new file mode 100644
index 0000000000..17ddf88a53
--- /dev/null
+++ b/cime_config/SystemTests/systemtest_utils.py
@@ -0,0 +1,57 @@
+"""
+Reduce code duplication by putting reused functions here.
+"""
+
+import os, subprocess
+
+
+def cmds_to_setup_conda(caseroot):
+ # Add specific commands needed on different machines to get conda available
+ # Use semicolon here since it's OK to fail
+ #
+ conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; "
+ # Execute the module unload/load when "which conda" fails
+ # eg on cheyenne
+ try:
+ subprocess.run("which conda", shell=True, check=True)
+ except subprocess.CalledProcessError:
+ # Remove python and add conda to environment for cheyennne
+ conda_setup_commands += " module unload python; module load conda;"
+
+ return conda_setup_commands
+
+
+def run_python_script(caseroot, this_conda_env, command, tool_path):
+
+ # Run in the specified conda environment
+ conda_setup_commands = cmds_to_setup_conda(caseroot)
+ conda_setup_commands += f" conda run -n {this_conda_env}"
+
+ # Finish with Python script call
+ command = conda_setup_commands + " " + command
+ print(f"command: {command}")
+
+ # Run with logfile
+ tool_name = os.path.split(tool_path)[-1]
+ try:
+ with open(tool_name + ".log", "w") as f:
+ subprocess.run(
+ command, shell=True, check=True, text=True, stdout=f, stderr=subprocess.STDOUT
+ )
+ except subprocess.CalledProcessError as error:
+ print("ERROR while getting the conda environment and/or ")
+ print(f"running the {tool_name} tool: ")
+ print(f"(1) If your {this_conda_env} environment is out of date or you ")
+ print(f"have not created the {this_conda_env} environment, yet, you may ")
+ print("get past this error by running ./py_env_create ")
+ print("in your ctsm directory and trying this test again. ")
+ print("(2) If conda is not available, install and load conda, ")
+ print("run ./py_env_create, and then try this test again. ")
+ print("(3) If (1) and (2) are not the issue, then you may be ")
+ print(f"getting an error within {tool_name} itself. ")
+ print("Default error message: ")
+ print(error.output)
+ raise
+ except:
+ print(f"ERROR trying to run {tool_name}.")
+ raise
diff --git a/cime_config/config_tests.xml b/cime_config/config_tests.xml
index 0307ee7ef5..536f79aeec 100644
--- a/cime_config/config_tests.xml
+++ b/cime_config/config_tests.xml
@@ -113,6 +113,16 @@ This defines various CTSM-specific system tests
$STOP_N
+
+ Generate prescribed maturity requirements, then test with them
+ 1
+ FALSE
+ FALSE
+ never
+ $STOP_OPTION
+ $STOP_N
+
+