diff --git a/Externals.cfg b/Externals.cfg index 2df854055e..7a4eeb4a6e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -23,7 +23,7 @@ required = True local_path = components/mosart protocol = git repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_47 +tag = mosart1_0_48 required = True [mizuRoute] diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index c2a32da4ac..f046172f7e 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -1687,11 +1687,6 @@ sub process_namelist_inline_logic { ############################################# setup_logic_rooting_profile($opts, $nl_flags, $definition, $defaults, $nl); - ############################################# - # namelist group: friction_velocity # - ############################################# - setup_logic_friction_vel($opts, $nl_flags, $definition, $defaults, $nl); - ############################# # namelist group: cngeneral # ############################# @@ -1711,6 +1706,11 @@ sub process_namelist_inline_logic { ############################################# setup_logic_canopyfluxes($opts, $nl_flags, $definition, $defaults, $nl); + ########################################################## + # namelist group: friction_velocity (after canopyfluxes) # + ########################################################## + setup_logic_friction_vel($opts, $nl_flags, $definition, $defaults, $nl); + ############################################# # namelist group: canopyhydrology_inparm # ############################################# @@ -3898,10 +3898,11 @@ sub setup_logic_rooting_profile { #------------------------------------------------------------------------------- sub setup_logic_friction_vel { - # + # Must be after canopyfluxes so that use_biomass_heat_storage will be set my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; - add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'zetamaxstable' ); + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'zetamaxstable', + 'use_biomass_heat_storage'=>$nl_flags->{'use_biomass_heat_storage'}, 'phys'=>$nl_flags->{'phys'} ); } #------------------------------------------------------------------------------- @@ -3926,6 +3927,11 @@ sub setup_logic_canopyfluxes { if ( &value_is_true($nl->get_value('use_biomass_heat_storage') ) && &value_is_true( $nl_flags->{'use_fates'}) ) { $log->fatal_error('use_biomass_heat_storage can NOT be set to true when fates is on'); } + if ( &value_is_true($nl->get_value('use_biomass_heat_storage')) ) { + $nl_flags->{'use_biomass_heat_storage'} = ".true."; + } else { + $nl_flags->{'use_biomass_heat_storage'} = ".false."; + } } #------------------------------------------------------------------------------- diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 69643041a4..41103b5259 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -210,9 +210,10 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 1.d-2 -2.0d00 -0.5d00 -0.5d00 +2.0d00 +2.0d00 +0.5d00 +0.5d00 .true. @@ -1055,10 +1056,11 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_16pfts_Irrig_CMIP6 lnd/clm2/surfdata_map/surfdata_0.125nldas2_hist_16pfts_Irrig_CMIP6_simyr2005_c190412.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230123.nc + lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_5x5_amazon_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/ctsm1.0.dev094-2-g633be0eb/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c200521.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc @@ -1112,12 +1114,12 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.ARCTIC.ne30x4_hist_78pft lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts_CMIP6_simyr2000_c200426.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c230123.nc @@ -1132,9 +1134,6 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_16pfts_Irrig_CMIP6_s lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc - lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850_c190303.nc @@ -1160,13 +1159,13 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c190214.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c190214.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c230123.nc - -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c190214.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1210,8 +1209,6 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/release-clm5.0.18/landuse.timeseries_48x96_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc -lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c170824.nc lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c170824.nc @@ -1232,7 +1229,7 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_48x96_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc +>lnd/clm2/surfdata_map/ctsm5.1.dev116/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c230123.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 784d3b5e0c..f270cbfea4 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -319,22 +319,22 @@ sub cat_and_create_namelistinfile { foreach my $driver ( "mct", "nuopc" ) { print " For $driver driver\n\n"; # configuration, structure, irrigate, verbose, clm_demand, ssp_rcp, test, sim_year, use_case - foreach my $options ( "-configuration nwp", - "-structure fast", - "-namelist '&a irrigate=.true./'", "-verbose", "-ssp_rcp SSP1-2.6", "-test", "-sim_year 1850", - "-namelist '&a use_lai_streams=.true.,use_soil_moisture_streams=.true./'", - "-use_case 1850_control", + foreach my $options ( "-res 0.9x1.25 -configuration nwp", + "-res 0.9x1.25 -structure fast", + "-res 0.9x1.25 -namelist '&a irrigate=.true./'", "-res 0.9x1.25 -verbose", "-res 0.9x1.25 -ssp_rcp SSP1-2.6", "-res 0.9x1.25 -test", "-res 0.9x1.25 -sim_year 1850", + "-res 0.9x1.25 -namelist '&a use_lai_streams=.true.,use_soil_moisture_streams=.true./'", + "-res 0.9x1.25 -use_case 1850_control", "-res 1x1pt_US-UMB -clm_usr_name 1x1pt_US-UMB -namelist '&a fsurdat=\"/dev/null\"/'", "-res 1x1_brazil", - "-clm_start_type startup", "-namelist '&a irrigate=.false./' -crop -bgc bgc", - "-envxml_dir . -infile myuser_nl_clm", - "-ignore_ic_date -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\"/' -bgc bgc -crop", - "-clm_start_type branch -namelist '&a nrevsn=\"thing.nc\",use_init_interp=T/'", - "-ignore_ic_date -clm_start_type startup -namelist '&a finidat=\"thing.nc\"/' -bgc bgc -crop", + "-res 0.9x1.25 -clm_start_type startup", "-namelist '&a irrigate=.false./' -crop -bgc bgc", + "-res 0.9x1.25 -infile myuser_nl_clm", + "-res 0.9x1.25 -ignore_ic_date -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\"/' -bgc bgc -crop", + "-res 0.9x1.25 -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\",use_init_interp=T/'", + "-res 0.9x1.25 -ignore_ic_date -clm_start_type startup -namelist '&a finidat=\"thing.nc\"/' -bgc bgc -crop", ) { my $file = $startfile; &make_env_run(); - my $base_options = "-res 0.9x1.25 -envxml_dir . -driver $driver"; + my $base_options = "-envxml_dir . -driver $driver"; if ( $driver eq "mct" ) { $base_options = "$base_options -lnd_frac $DOMFILE"; } else { diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 899ab1aead..99a653129e 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -40,8 +40,11 @@ def __init__(self, case): self._cfg_file_path = os.path.join(self._get_caseroot(), 'modify_fsurdat.cfg') + logger.info(" create config file to modify") self._create_config_file() + logger.info(" run modify_fsurdat") self._run_modify_fsurdat() + logger.info(" modify user_nl files") self._modify_user_nl() with open('done_FSURDATMODIFYCTSM_setup.txt', 'w') as fp: pass @@ -68,7 +71,7 @@ def _run_modify_fsurdat(self): # Need to specify a specific python version that has the required # dependencies python_path = _get_python_path() - subprocess.check_call([python_path, tool_path, self._cfg_file_path]) + subprocess.check_call([python_path, tool_path, self._cfg_file_path, "--verbose", "--overwrite", "--allow_ideal_and_include_non_veg"]) def _modify_user_nl(self): append_to_user_nl_files(caseroot = self._get_caseroot(), diff --git a/doc/ChangeLog b/doc/ChangeLog index 49f49c235f..b1f5fdf975 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,161 @@ =============================================================== +Tag name: ctsm5.1.dev116 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Thu Jan 26 02:17:27 MST 2023 +One-line Summary: Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files + +Purpose and description of changes +---------------------------------- + +Change zetamaxstable to 2 when biomass-heat-storage is on. This changes simulation answers once they +run long enough to exceed that threshold. + +Also fix an issue with maintence respiration (MR) for BGC simulations. This changes answers for most BGC cases once +they run long enough. Live course MR wasn't included. + +Make the default for MOSART to send negative flow to river outlets. Also fix an issue with this mode. + +Bring in new surface datasets for the single point sites. We now make these sites using subset_data rather +than mksurfdata. + +Some new capability to the subset_data and modify_fsurdat tools. + +subset_data add options: +--out-surface -- To name the surface dataset on the command line rather than based on the current date +--cfg-file ----- Enter the default configure file to use rather than assume a fixed one + +modify_fsurdat add options: +--fsurdat_in -- to input on command line rather than config file +--fsurdat_out -- to input on command line rather than config file +--allow_ideal_and_include_non_veg -- to allow idealized and include_non_veg at the same time +--allow_dom_pft_and_idealized -- to allow dom_pft and idealized at the same time +--overwrite -- allow output file to be overwritten +config file options: +process_subgrid_section -- Read in an optional section to set the PCT_* fractions +process_var_list_section - Read in an optional section to set any variable on the file + +Add --silent option to python tools. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[X] clm5_1 + +[x] clm5_0 + +[ ] ctsm5_0-nwp + +[x] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + Fixes #1676 -- live coarse maintenance respiration is not included in the root respiration + Fixes #1674 -- Change mksurfdata_map/mksurfdata_esmf Makefile to build single-point datasets using subset_data + Fixes #1809 -- Add ability to name a different default config file for subset_data + Fixes #1941 -- Add --silent option to ctsm_logging python infrastructure + Fixes #1942 -- Move py_env_create outside of tools test driver, as fails on compute nodes on cheyenne + Fixes #1924 -- Some updates to fsurdat_modifier script + Fixes #1690 -- Set and use zetamaxstable for BHS cases + Fixes #1689 -- Set zetamaxstable to 2 consistently for BHS + +Externals issues fixed (include issue #): + MOSART #58 Make negative and direct_to_outlet the default option + MOSART #56 Some issues with direct_to_outlet + +Notes of particular relevance for users +--------------------------------------- + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): tools + Just to the subset_data and fsurdat_modifier tools as outlined above + +Changes made to namelist defaults (e.g., changed parameter values): + zetamaxstable now 2.0 is use_biomass_heat_storage is on + +Changes to the datasets (e.g., parameter, surface or initial files): Single point fsurdat + 1x1_brazil, 1x1_numaIA, 1x1_smallvilleIA, 1x1_vancouverCAN, 1x1_mexicocityMEX, 1x1_urbanc_alpha + surface datasets and 1x1_brazil landuse.timeseries + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + tools test now just activates the ctsm_py conda environment rather than creating it + There are seperate configure files for each urban surface dataset for modify_fsurdat + There is a 1850 configure file for subset_data + +Changes to tests or testing: + fsurdatmodifyctsm.py script adjusted to compensate for changes + python directory changes include unit and system tests to support updates + Single point mksurdata_map tests were removed + + +Testing summary: regular tools +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - PASS + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - PASS + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + cheyenne - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + +Answer changes +-------------- + +Changes answers relative to baseline: + + Summarize any changes to answers, i.e., + - what code configurations: clm5_1, supported single point resolutions + - what platforms/compilers: All + - nature of change: adjusted climate + clm5_1 changes because zetmaxstable set to 2.0. If a simulation runs long enough + this max will be hit and it will change answers once it does. But if stability + doesn't hit the max answers can be identical. + clm5_0 and clm4_5 also change if biomass heat storage is turned on + single point resolutions (i.e. 1x1_smallvilleIA, 1x1_brazil, 1x1_mexicocityMEX) have differences + maintenence respiration + + If this tag changes climate describe the run(s) done to evaluate the new + climate (put details of the simulations in the experiment database) + Keith Oleson ran experiments with changing zetamaxstable some slides showing this are here: + https://docs.google.com/presentation/d/1u6ycr7F97QYYRcRfEdD9yIxH75diUx2r + + +Other details +------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): mosart + mosart updated to mosart1_0_48 + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + #1812 -- Get single point surface datasets from subset_data rather than mksurfdata + #1802 -- Make zetamaxstable consistently 2.0 when BHS on + #1915 -- Fix issue #1864 in release documentation + Update manage_externals (direct push to main-dev) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev115 Originator(s): rgknox (Ryan Knox) Date: Fri Dec 2 15:45:32 MST 2022 diff --git a/doc/ChangeSum b/doc/ChangeSum index 1d42a3522d..3042b38a2a 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements diff --git a/doc/design/python_script_user_interface.rst b/doc/design/python_script_user_interface.rst index 4dd0c9c546..3ad6a4d2cf 100644 --- a/doc/design/python_script_user_interface.rst +++ b/doc/design/python_script_user_interface.rst @@ -40,8 +40,8 @@ Options that are more than a single character should be formatted as ``--some-va 2. Standard options: verbose, silent, help and debug: -Scripts should support ``--verbose`` / ``-v`` and ``--debug`` options. See comments at the top of ``ctsm_logging.py`` for details. -Also the options silent and help are recommended as well. +Scripts should support ``--verbose`` / ``-v``, ''--silent'', and ``--debug`` options. See comments at the top of ``ctsm_logging.py`` for details. +Also the help option is highly recommended as well. 3. Value flags: diff --git a/python/Makefile b/python/Makefile index bf219f16b6..a1eaba0928 100644 --- a/python/Makefile +++ b/python/Makefile @@ -24,6 +24,10 @@ PYLINT_SRC = \ ctsm all: test lint black + @echo + @echo + @echo "Successfully ran all standard tests" + test: utest stest .PHONY: utest diff --git a/python/ctsm/config_utils.py b/python/ctsm/config_utils.py index 847a1804f6..bd53825f14 100644 --- a/python/ctsm/config_utils.py +++ b/python/ctsm/config_utils.py @@ -86,6 +86,40 @@ def get_config_value( return val +def get_config_value_or_array( + config, + section, + item, + convert_to_type=None, +): + """Get a config value as a single value or as an array if it's expressed as an array + for cases when you don't know how it's going to be expressed""" + val = config.get(section, item) + vallist = val.split() + if convert_to_type is not None: + if ( + convert_to_type is not float + and convert_to_type is not int + and convert_to_type is not str + ): + abort( + "get_config_value_or_array can only have convert_to_type as float, int or str not " + + str(convert_to_type) + ) + is_list = bool(len(vallist) > 1) + + val = _handle_config_value( + var=val, + default=None, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=False, + allowed_values=None, + ) + return val + + def _handle_config_value( var, default, item, is_list, convert_to_type, can_be_unset, allowed_values ): diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index ff51c6d8f2..e14ec2754c 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -68,6 +68,7 @@ def add_logging_args(parser): logging_level.add_argument( "-v", "--verbose", action="store_true", help="Output extra logging info" ) + logging_level.add_argument("--silent", action="store_true", help="Only output errors") logging_level.add_argument( "--debug", @@ -84,6 +85,8 @@ def process_logging_args(args): root_logger.setLevel(logging.DEBUG) elif args.verbose: root_logger.setLevel(logging.INFO) + elif args.silent: + root_logger.setLevel(logging.ERROR) else: root_logger.setLevel(logging.WARNING) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 7b7abbc646..75fbbdba12 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -11,7 +11,7 @@ from configparser import ConfigParser from ctsm.utils import abort, write_output -from ctsm.config_utils import get_config_value +from ctsm.config_utils import get_config_value, get_config_value_or_array from ctsm.ctsm_logging import ( setup_logging_pre_config, add_logging_args, @@ -29,112 +29,406 @@ def main(): Calls function that modifies an fsurdat (surface dataset) """ + args = fsurdat_modifier_arg_process() + fsurdat_modifier(args) + + +def fsurdat_modifier_arg_process(): + """Argument processing for fsurdat_modifier script""" # set up logging allowing user control setup_logging_pre_config() # read the command line argument to obtain the path to the .cfg file parser = argparse.ArgumentParser() parser.add_argument("cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg") + parser.add_argument( + "-i", + "--fsurdat_in", + default="UNSET", + required=False, + type=str, + help="The input surface dataset to modify. ", + ) + parser.add_argument( + "-o", + "--fsurdat_out", + required=False, + default="UNSET", + type=str, + help="The output surface dataset with the modifications. ", + ) + parser.add_argument( + "--allow_ideal_and_include_non_veg", + required=False, + default=False, + action="store_true", + help="Allow both idealized and include_nonveg to be on at the same time. ", + ) + parser.add_argument( + "--allow_dom_pft_and_idealized", + required=False, + default=False, + action="store_true", + help="Allow both idealized and dom_pft to be on at the same time. ", + ) + parser.add_argument( + "--overwrite", + required=False, + default=False, + action="store_true", + help="Overwrite the output file if it already exists. ", + ) add_logging_args(parser) args = parser.parse_args() process_logging_args(args) - fsurdat_modifier(args.cfg_path) + # Error checking of arguments + if not os.path.exists(args.cfg_path): + abort("Config file does NOT exist: " + str(args.cfg_path)) + return args -def fsurdat_modifier(cfg_path): - """Implementation of fsurdat_modifier command""" - # read the .cfg (config) file - config = ConfigParser() - config.read(cfg_path) - section = config.sections()[0] # name of the first section - # required: user must set these in the .cfg file - fsurdat_in = get_config_value( - config=config, section=section, item="fsurdat_in", file_path=cfg_path - ) - fsurdat_out = get_config_value( - config=config, section=section, item="fsurdat_out", file_path=cfg_path - ) +def check_no_subgrid_section(config): + """Check that there isn't a subgrid section when it's processing is turned off""" + section = "modify_fsurdat_subgrid_fractions" + if config.has_section(section): + abort( + "Config file does have a section: " + + section + + " that should NOT be there since it is turned off" + ) - # required but fallback values available for variables omitted - # entirely from the .cfg file - idealized = get_config_value( + +def check_no_varlist_section(config): + """Check that there isn't a var list section when it's processing is turned off""" + section = "modify_fsurdat_variable_list" + if config.has_section(section): + abort( + "Config file does have a section: " + + section + + " that should NOT be there since it is turned off" + ) + + +def check_range(var, section, value, minval, maxval): + """Check that the value is within range""" + if value < minval or value > maxval: + abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + str(value)) + + +def read_cfg_subgrid(config, cfg_path, numurbl=3): + """Read the subgrid fraction section from the config file""" + section = "modify_fsurdat_subgrid_fractions" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) + + subgrid_settings = {} + var_list = config.options(section) + valid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] + varsum = 0 + for var in var_list: + if valid_list.count(var) == 0: + abort( + "Variable " + + var + + " in " + + section + + " is not a valid variable name. Valid vars =" + + str(valid_list) + ) + # Urban is multidimensional + if var == "pct_urban": + vallist = get_config_value( + config=config, + section=section, + item=var, + file_path=cfg_path, + is_list=True, + convert_to_type=float, + ) + if len(vallist) != numurbl: + abort("PCT_URBAN is not a list of the expected size of " + str(numurbl)) + # so if a scalar value, must be multiplied # by the density dimension + for val in vallist: + check_range(var, section, val, 0.0, 100.0) + varsum += val + value = vallist + else: + value = get_config_value( + config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float + ) + check_range(var, section, value, 0.0, 100.0) + varsum += value + + subgrid_settings[var.upper()] = value + + if varsum != 100.0: + abort( + "PCT fractions in subgrid section do NOT sum to a hundred as they should. Sum = " + + str(varsum) + ) + + return subgrid_settings + + +def read_cfg_var_list(config, idealized=True): + """Read the variable list section from the config file""" + section = "modify_fsurdat_variable_list" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) + + varlist_settings = {} + var_list = config.options(section) + ideal_list = [ + "soil_color", + "pct_sand", + "pct_clay", + "organic", + "pct_cft", + "pct_nat_pft", + "fmax", + "std_elev", + ] + subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] + # List of variables that should be excluded because they are changed elsewhere, + # or they shouldn't be changed # Ds, Dsmax, and Ws are excluded because they + # are of mixed case and we only search for varaibles in lowercase + # or uppercase and not mixed case. + monthly_list = [ + "monthly_lai", + "monthly_sai", + "monthly_height_top", + "monthly_height_bot", + "ds", + "mxsoil_color", + "natpft", + "cft", + "time", + "longxy", + "latixy", + "dsmax", + "area", + "ws", + ] + for var in var_list: + if idealized and ideal_list.count(var) != 0: + abort( + var + + " is a special variable handled in the idealized section." + + " This should NOT be handled in the variiable list section." + + " Special idealized vars =" + + str(ideal_list) + ) + if subgrid_list.count(var) != 0: + abort( + var + + " is a variable handled in the subgrid section." + + " This should NOT be handled in the variiable list section." + + " Subgrid vars =" + + str(subgrid_list) + ) + if monthly_list.count(var) != 0: + abort( + var + + " is a variable handled as part of the dom_pft handling." + + " This should NOT be handled in the variiable list section." + + " Monthly vars handled this way =" + + str(monthly_list) + ) + value = get_config_value_or_array( + config=config, section=section, item=var, convert_to_type=float + ) + varlist_settings[var] = value + + return varlist_settings + + +def modify_optional( + modify_fsurdat, + idealized, + include_nonveg, + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, +): + """Modify the dataset according to the optional settings""" + + # Set fsurdat variables in a rectangle that could be global (default). + # Note that the land/ocean mask gets specified in the domain file for + # MCT or the ocean mesh files for NUOPC. Here the user may specify + # fsurdat variables inside a box but cannot change which points will + # run as land and which as ocean. + if idealized: + modify_fsurdat.set_idealized() # set 2D variables + # set 3D and 4D variables pertaining to natural vegetation + modify_fsurdat.set_dom_pft(dom_pft=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) + logger.info("idealized complete") + + if max_sat_area is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("FMAX", max_sat_area) + logger.info("max_sat_area complete") + + if std_elev is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("STD_ELEV", std_elev) + logger.info("std_elev complete") + + if soil_color is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("SOIL_COLOR", soil_color) + logger.info("soil_color complete") + + if not include_nonveg: + modify_fsurdat.zero_nonveg() + logger.info("zero_nonveg complete") + + # set_dom_pft follows zero_nonveg because it modifies PCT_NATVEG + # and PCT_CROP in the user-defined rectangle + if dom_pft is not None: + modify_fsurdat.set_dom_pft( + dom_pft=dom_pft, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot + ) + logger.info("dom_pft complete") + + +def read_cfg_optional_basic_opts(modify_fsurdat, config, cfg_path, section): + """Read the optional parts of the main section of the config file. + The main section is called modify_fsurdat_basic_options. + Users may set these optional parts but are not required to do so.""" + + lai = get_config_value( config=config, section=section, - item="idealized", + item="lai", file_path=cfg_path, - convert_to_type=bool, + is_list=True, + convert_to_type=float, + can_be_unset=True, ) - include_nonveg = get_config_value( + sai = get_config_value( config=config, section=section, - item="include_nonveg", + item="sai", file_path=cfg_path, - convert_to_type=bool, + is_list=True, + convert_to_type=float, + can_be_unset=True, ) - - lnd_lat_1 = get_config_value( + hgt_top = get_config_value( config=config, section=section, - item="lnd_lat_1", + item="hgt_top", file_path=cfg_path, + is_list=True, convert_to_type=float, + can_be_unset=True, ) - lnd_lat_2 = get_config_value( + hgt_bot = get_config_value( config=config, section=section, - item="lnd_lat_2", + item="hgt_bot", file_path=cfg_path, + is_list=True, convert_to_type=float, + can_be_unset=True, ) - lnd_lon_1 = get_config_value( + + max_soil_color = int(modify_fsurdat.file.mxsoil_color) + soil_color = get_config_value( config=config, section=section, - item="lnd_lon_1", + item="soil_color", file_path=cfg_path, - convert_to_type=float, + allowed_values=range(1, max_soil_color + 1), # 1 to max_soil_color + convert_to_type=int, + can_be_unset=True, ) - lnd_lon_2 = get_config_value( + + std_elev = get_config_value( config=config, section=section, - item="lnd_lon_2", + item="std_elev", file_path=cfg_path, convert_to_type=float, + can_be_unset=True, ) - - landmask_file = get_config_value( + max_sat_area = get_config_value( config=config, section=section, - item="landmask_file", + item="max_sat_area", file_path=cfg_path, + convert_to_type=float, can_be_unset=True, ) + return ( + max_sat_area, + std_elev, + soil_color, + lai, + sai, + hgt_top, + hgt_bot, + ) - lat_dimname = get_config_value( - config=config, section=section, item="lat_dimname", file_path=cfg_path, can_be_unset=True + +def read_cfg_option_control( + modify_fsurdat, + config, + section, + cfg_path, + allow_ideal_and_include_non_veg=False, + allow_dom_pft_and_idealized=False, +): + """Read the option control section""" + # required but fallback values available for variables omitted + # entirely from the .cfg file + idealized = get_config_value( + config=config, + section=section, + item="idealized", + file_path=cfg_path, + convert_to_type=bool, ) - lon_dimname = get_config_value( - config=config, section=section, item="lon_dimname", file_path=cfg_path, can_be_unset=True + if idealized: + logger.info("idealized option is on") + else: + logger.info("idealized option is off") + process_subgrid = get_config_value( + config=config, + section=section, + item="process_subgrid_section", + file_path=cfg_path, + convert_to_type=bool, ) - - # Create ModifyFsurdat object - modify_fsurdat = ModifyFsurdat.init_from_file( - fsurdat_in, - lnd_lon_1, - lnd_lon_2, - lnd_lat_1, - lnd_lat_2, - landmask_file, - lat_dimname, - lon_dimname, + if process_subgrid: + logger.info("process_subgrid_section option is on") + else: + logger.info("process_subgrid_section option is off") + process_var_list = get_config_value( + config=config, + section=section, + item="process_var_list_section", + file_path=cfg_path, + convert_to_type=bool, ) - - # If output file exists, abort before starting work - if os.path.exists(fsurdat_out): - errmsg = "Output file already exists: " + fsurdat_out - abort(errmsg) - - # not required: user may set these in the .cfg file + if process_var_list: + logger.info("process_var_list_section option is on") + else: + logger.info("process_var_list_section option is off") + include_nonveg = get_config_value( + config=config, + section=section, + item="include_nonveg", + file_path=cfg_path, + convert_to_type=bool, + ) + if include_nonveg: + logger.info("include_nonveg option is on") + else: + logger.info("include_nonveg option is off") max_pft = int(max(modify_fsurdat.file.lsmpft)) dom_pft = get_config_value( config=config, @@ -145,110 +439,193 @@ def fsurdat_modifier(cfg_path): convert_to_type=int, can_be_unset=True, ) + if dom_pft: + logger.info("dom_pft option is on and = %s", str(dom_pft)) + else: + logger.info("dom_pft option is off") + if dom_pft is not None and idealized and not allow_dom_pft_and_idealized: + abort("idealized AND dom_pft can NOT both be on, pick one or the other") + if include_nonveg and idealized and not allow_ideal_and_include_non_veg: + abort("idealized AND include_nonveg can NOT both be on, pick one or the other") + if process_subgrid and idealized: + abort("idealized AND process_subgrid_section can NOT both be on, pick one or the other") - lai = get_config_value( + return (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) + + +def read_cfg_required_basic_opts(config, section, cfg_path): + """Read the required part of the control section""" + lnd_lat_1 = get_config_value( config=config, section=section, - item="lai", + item="lnd_lat_1", file_path=cfg_path, - is_list=True, convert_to_type=float, - can_be_unset=True, ) - sai = get_config_value( + lnd_lat_2 = get_config_value( config=config, section=section, - item="sai", + item="lnd_lat_2", file_path=cfg_path, - is_list=True, convert_to_type=float, - can_be_unset=True, ) - hgt_top = get_config_value( + lnd_lon_1 = get_config_value( config=config, section=section, - item="hgt_top", + item="lnd_lon_1", file_path=cfg_path, - is_list=True, convert_to_type=float, - can_be_unset=True, ) - hgt_bot = get_config_value( + lnd_lon_2 = get_config_value( config=config, section=section, - item="hgt_bot", + item="lnd_lon_2", file_path=cfg_path, - is_list=True, convert_to_type=float, - can_be_unset=True, ) - max_soil_color = int(modify_fsurdat.file.mxsoil_color) - soil_color = get_config_value( + landmask_file = get_config_value( config=config, section=section, - item="soil_color", + item="landmask_file", file_path=cfg_path, - allowed_values=range(1, max_soil_color + 1), # 1 to max_soil_color - convert_to_type=int, can_be_unset=True, ) - std_elev = get_config_value( - config=config, - section=section, - item="std_elev", - file_path=cfg_path, - convert_to_type=float, - can_be_unset=True, + lat_dimname = get_config_value( + config=config, section=section, item="lat_dimname", file_path=cfg_path, can_be_unset=True ) - max_sat_area = get_config_value( - config=config, - section=section, - item="max_sat_area", - file_path=cfg_path, - convert_to_type=float, - can_be_unset=True, + lon_dimname = get_config_value( + config=config, section=section, item="lon_dimname", file_path=cfg_path, can_be_unset=True ) + return (lnd_lat_1, lnd_lat_2, lnd_lon_1, lnd_lon_2, landmask_file, lat_dimname, lon_dimname) - # ------------------------------ - # modify surface data properties - # ------------------------------ - # Set fsurdat variables in a rectangle that could be global (default). - # Note that the land/ocean mask gets specified in the domain file for - # MCT or the ocean mesh files for NUOPC. Here the user may specify - # fsurdat variables inside a box but cannot change which points will - # run as land and which as ocean. - if idealized: - modify_fsurdat.set_idealized() # set 2D variables - # set 3D and 4D variables pertaining to natural vegetation - modify_fsurdat.set_dom_pft(dom_pft=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) - logger.info("idealized complete") +def fsurdat_modifier(parser): + """Implementation of fsurdat_modifier command""" + # read the .cfg (config) file + cfg_path = str(parser.cfg_path) + config = ConfigParser() + config.read(cfg_path) + section = "modify_fsurdat_basic_options" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) - if max_sat_area is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("FMAX", max_sat_area) - logger.info("max_sat_area complete") + if parser.fsurdat_in == "UNSET": + # required: user must set these in the .cfg file + fsurdat_in = get_config_value( + config=config, section=section, item="fsurdat_in", file_path=cfg_path + ) + else: + if config.has_option(section=section, option="fsurdat_in"): + abort("fsurdat_in is specified in both the command line and the config file, pick one") + fsurdat_in = str(parser.fsurdat_in) - if std_elev is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("STD_ELEV", std_elev) - logger.info("std_elev complete") + # Error checking of input file + if not os.path.exists(fsurdat_in): + abort("Input fsurdat_in file does NOT exist: " + str(fsurdat_in)) - if soil_color is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("SOIL_COLOR", soil_color) - logger.info("soil_color complete") + if parser.fsurdat_out == "UNSET": + fsurdat_out = get_config_value( + config=config, section=section, item="fsurdat_out", file_path=cfg_path + ) + else: + if config.has_option(section=section, option="fsurdat_out"): + abort("fsurdat_out is specified in both the command line and the config file, pick one") + fsurdat_out = str(parser.fsurdat_out) - if not include_nonveg: - modify_fsurdat.zero_nonveg() - logger.info("zero_nonveg complete") + # If output file exists, abort before starting work + if os.path.exists(fsurdat_out): + if not parser.overwrite: + errmsg = "Output file already exists: " + fsurdat_out + abort(errmsg) + else: + warnmsg = ( + "Output file already exists" + + ", but the overwrite option was selected so the file will be overwritten." + ) + logger.warning(warnmsg) + ( + lnd_lat_1, + lnd_lat_2, + lnd_lon_1, + lnd_lon_2, + landmask_file, + lat_dimname, + lon_dimname, + ) = read_cfg_required_basic_opts(config, section, cfg_path) + # Create ModifyFsurdat object + modify_fsurdat = ModifyFsurdat.init_from_file( + fsurdat_in, + lnd_lon_1, + lnd_lon_2, + lnd_lat_1, + lnd_lat_2, + landmask_file, + lat_dimname, + lon_dimname, + ) - # set_dom_pft follows zero_nonveg because it modifies PCT_NATVEG - # and PCT_CROP in the user-defined rectangle - if dom_pft is not None: - modify_fsurdat.set_dom_pft( - dom_pft=dom_pft, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot - ) - logger.info("dom_pft complete") + # Read control information about the optional sections + ( + idealized, + process_subgrid, + process_var_list, + include_nonveg, + dom_pft, + ) = read_cfg_option_control( + modify_fsurdat, + config, + section, + cfg_path, + parser.allow_ideal_and_include_non_veg, + parser.allow_dom_pft_and_idealized, + ) + + # Read parts that are optional + ( + max_sat_area, + std_elev, + soil_color, + lai, + sai, + hgt_top, + hgt_bot, + ) = read_cfg_optional_basic_opts(modify_fsurdat, config, cfg_path, section) + # ------------------------------ + # modify surface data properties + # ------------------------------ + + modify_optional( + modify_fsurdat, + idealized, + include_nonveg, + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, + ) + # + # Handle optional sections + # + if process_subgrid: + subgrid = read_cfg_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) + modify_fsurdat.set_varlist(subgrid, cfg_path) + logger.info("process_subgrid is complete") + else: + check_no_subgrid_section(config) + + if process_var_list: + varlist = read_cfg_var_list(config, idealized=idealized) + update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) + modify_fsurdat.set_varlist(update_list, cfg_path) + logger.info("process_var_list is complete") + else: + check_no_varlist_section(config) # ---------------------------------------------- # Output the now modified CTSM surface data file diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index ba1fd18fdb..599b8a6e84 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -29,7 +29,12 @@ def __init__( self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file, lat_dimname, lon_dimname ): + self.numurbl = 3 # Number of urban density types self.file = my_data + if "numurbl" in self.file.dims: + self.numurbl = self.file.dims["numurbl"] + else: + abort("numurbl is not a dimension on the input surface dataset file and needs to be") self.rectangle = self._get_rectangle( lon_1=lon_1, @@ -115,6 +120,10 @@ def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): return rectangle + def get_urb_dens(self): + """Get the number of urban density classes""" + return self.numurbl + def write_output(self, fsurdat_in, fsurdat_out): """ Description @@ -214,6 +223,96 @@ def set_dom_pft(self, dom_pft, lai, sai, hgt_top, hgt_bot): if val is not None: self.set_lai_sai_hgts(dom_pft=dom_pft, var=var, val=val) + def check_varlist(self, settings, allow_uppercase_vars=False): + """ + Check a list of variables from a dictionary of settings + """ + settings_return = {} + varlist = settings.keys() + for var in varlist: + varname = var + val = settings[varname] + if not var in self.file: + if not allow_uppercase_vars: + errmsg = "Error: Variable " + varname + " is NOT in the file" + abort(errmsg) + if not varname.upper() in self.file: + errmsg = "Error: Variable " + varname.upper() + " is NOT in the file" + abort(errmsg) + varname = varname.upper() + + settings_return[varname] = val + # + # Check that dimensions are as expected + # + if len(self.file[varname].dims) == 2: + if not isinstance(val, float): + abort( + "For 2D vars, there should only be a single value for variable = " + varname + ) + elif len(self.file[varname].dims) >= 3: + dim1 = int(self.file.sizes[self.file[varname].dims[0]]) + if not isinstance(val, list): + abort( + "For higher dimensional vars, the variable needs to be expressed " + + "as a list of values of the dimension size = " + + str(dim1) + + " for variable=" + + varname + ) + if len(val) != dim1: + abort( + "Variable " + varname + " is of the wrong size. It should be = " + str(dim1) + ) + return settings_return + + def set_varlist(self, settings, cfg_path="unknown-config-file"): + """ + Set a list of variables from a dictionary of settings + """ + for var in settings.keys(): + if var in self.file: + if len(self.file[var].dims) == 2: + if not isinstance(settings[var], float): + abort( + "For 2D vars, there should only be a single value for variable = " + var + ) + self.setvar_lev0(var, settings[var]) + elif len(self.file[var].dims) == 3: + dim1 = int(self.file.sizes[self.file[var].dims[0]]) + vallist = settings[var] + if not isinstance(vallist, list): + abort( + "For higher dimensional vars, there must be a list of values " + + "for variable= " + + var + + " from the config file = " + + cfg_path + ) + if len(vallist) != dim1: + abort( + "Variable " + var + " is of the wrong size. It should be = " + str(dim1) + ) + for lev1 in range(dim1): + self.setvar_lev1(var, vallist[lev1], lev1_dim=lev1) + elif len(self.file[var].dims) == 4: + dim_lev1 = int(self.file.sizes[self.file[var].dims[1]]) + dim_lev2 = int(self.file.sizes[self.file[var].dims[0]]) + vallist = settings[var] + for lev1 in range(dim_lev1): + for lev2 in range(dim_lev2): + self.setvar_lev2(var, vallist[lev2], lev1_dim=lev1, lev2_dim=lev2) + else: + abort( + "Error: Variable " + + var + + " is a higher dimension than currently allowed = " + + str(self.file[var].dims) + ) + else: + errmsg = "Error: Variable " + var + " is NOT in the file" + abort(errmsg) + def set_lai_sai_hgts(self, dom_pft, var, val): """ Description diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index 8d5d81e014..2a2dc66bce 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -145,7 +145,7 @@ def create_domain_at_reg(self, indir, file): f_in.close() f_out.close() - def create_surfdata_at_reg(self, indir, file, user_mods_dir): + def create_surfdata_at_reg(self, indir, file, user_mods_dir, specify_fsurf_out): """ Create surface data file for this RegionalCase class. """ @@ -154,7 +154,11 @@ def create_surfdata_at_reg(self, indir, file, user_mods_dir): # specify files fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + if specify_fsurf_out is None: + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + else: + fsurf_out = specify_fsurf_out + logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index 05b918d316..96544ff9c1 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -450,7 +450,7 @@ def modify_surfdata_atpoint(self, f_orig): return f_mod - def create_surfdata_at_point(self, indir, file, user_mods_dir): + def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out): """ Create surface data file at a single point. """ @@ -464,7 +464,10 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir): # specify file fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + if specify_fsurf_out is None: + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + else: + fsurf_out = specify_fsurf_out logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 34a9e583d0..4a3a5801f1 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -76,7 +76,7 @@ process_logging_args, ) -DEFAULTS_FILE = "default_data.cfg" +DEFAULTS_CONFIG = "tools/site_and_regional/default_data.cfg" logger = logging.getLogger(__name__) @@ -309,6 +309,26 @@ def get_parser(): type=str, default="", ) + + subparser.add_argument( + "--out-surface", + help="Output surface dataset name \ + (if you want to override the default based on the current date). \n \ + (only valid if outputing a surface dataset)", + action="store", + dest="out_surface", + type=str, + ) + cesmroot = path_to_ctsm_root() + defaults_file = os.path.join(cesmroot, DEFAULTS_CONFIG) + subparser.add_argument( + "--cfg-file", + help="Default configure file to use for default filenames.", + action="store", + dest="config_file", + type=str, + default=defaults_file, + ) subparser.add_argument( "--overwrite", help="Flag to overwrite if the files already exists.", @@ -335,6 +355,64 @@ def get_parser(): return parser +def check_args(args): + """Check the command line arguments""" + # --------------------------------- # + # print help and exit when no option is chosen + if args.run_type not in ("point", "region"): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Must supply a positional argument: 'point' or 'region'. + """ + ) + raise argparse.ArgumentError(None, err_msg) + + if not any( + [ + args.create_surfdata, + args.create_domain, + args.create_landuse, + args.create_datm, + ] + ): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Must supply one of: + \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n + """ + ) + raise argparse.ArgumentError(None, err_msg) + + if not os.path.exists(args.config_file): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Entered default config file does not exist" + """ + ) + raise argparse.ArgumentError(None, err_msg) + + if args.out_surface and not args.create_surfdata: + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n out-surface option is given without the --create-surface option" + """ + ) + raise argparse.ArgumentError(None, err_msg) + + if args.out_surface and os.path.exists(args.out_surface) and not args.overwrite: + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n out-surface filename exists and the overwrite option was not also selected" + """ + ) + raise argparse.ArgumentError(None, err_msg) + + def setup_user_mods(user_mods_dir, cesmroot): """ Sets up the user mods files and directories @@ -414,6 +492,10 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") + if args.out_surface: + fsurf_out = args.out_surface + else: + fsurf_out = None file_dict = { "main_dir": clmforcingindir, @@ -427,6 +509,7 @@ def setup_files(args, defaults, cesmroot): os.path.join(defaults.get("landuse", "dir")), ), "fsurf_in": fsurf_in, + "fsurf_out": fsurf_out, "fluse_in": fluse_in, "datm_tuple": DatmFiles( dir_input_datm, @@ -486,7 +569,10 @@ def subset_point(args, file_dict: dict): # -- Create CTSM surface data file if single_point.create_surfdata: single_point.create_surfdata_at_point( - file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + file_dict["fsurf_dir"], + file_dict["fsurf_in"], + args.user_mods_dir, + specify_fsurf_out=file_dict["fsurf_out"], ) # -- Create CTSM transient landuse data file @@ -546,7 +632,10 @@ def subset_region(args, file_dict: dict): # -- Create CTSM surface data file if region.create_surfdata: region.create_surfdata_at_reg( - file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + file_dict["fsurf_dir"], + file_dict["fsurf_in"], + args.user_mods_dir, + specify_fsurf_out=file_dict["fsurf_out"], ) # -- Create CTSM transient landuse data file @@ -570,34 +659,7 @@ def main(): parser = get_parser() args = parser.parse_args() - # --------------------------------- # - # print help and exit when no option is chosen - if args.run_type != "point" and args.run_type != "region": - err_msg = textwrap.dedent( - """\ - \n ------------------------------------ - \n Must supply a positional argument: 'point' or 'region'. - """ - ) - raise parser.error(err_msg) - - if not any( - [ - args.create_surfdata, - args.create_domain, - args.create_landuse, - args.create_datm, - ] - ): - err_msg = textwrap.dedent( - """\ - \n ------------------------------------ - \n Must supply one of: - \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n - """ - ) - raise parser.error(err_msg) - + check_args(args) # --------------------------------- # # process logging args (i.e. debug and verbose) process_logging_args(args) @@ -606,7 +668,7 @@ def main(): # parse defaults file cesmroot = path_to_ctsm_root() defaults = configparser.ConfigParser() - defaults.read(os.path.join(cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) + defaults.read(args.config_file) # --------------------------------- # myname = getuser() diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 6ee00604be..cee7a51d36 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -10,12 +10,15 @@ import unittest import tempfile import shutil +import sys import xarray as xr +import numpy as np from ctsm.path_utils import path_to_ctsm_root from ctsm import unit_testing from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier +from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -39,6 +42,7 @@ def setUp(self): path_to_ctsm_root(), "tools/modify_input_files/modify_fsurdat_template.cfg" ) testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._testinputs_path = testinputs_path self._fsurdat_in = os.path.join( testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", @@ -53,6 +57,264 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) + def test_no_files_given_fail(self): + """ + Test that if no input or output files are given that it will gracefully fail + """ + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex(SystemExit, "must contain item 'fsurdat_in'"): + fsurdat_modifier(parser) + + def test_short_config(self): + """ + Test that a short config file works + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_out = ( + "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" + ) + if os.path.exists(fsurdat_out): + os.remove(fsurdat_out) + fsurdat_modifier(parser) + # Run it again with the overwrite option so that it will overwrite the file just created + sys.argv = ["fsurdat_modifier", self._cfg_file_path, "--overwrite"] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + # Cleanup + os.remove(fsurdat_out) + + def test_short_infile_both_cmdline_and_cfg(self): + """ + Test that a graceful fail happens when the infile + is given both in the command line and the config file + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + "specify_fsurdat_in_on_cmd_line.nc", + ] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex( + SystemExit, + "fsurdat_in is specified in both the command line and the config file, pick one", + ): + fsurdat_modifier(parser) + + def test_short_outfile_both_cmdline_and_cfg(self): + """ + Test that a graceful fail happens when the outfile is given + both in the command line and the config file + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-o", + "specify_fsurdat_out_on_cmd_line.nc", + ] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex( + SystemExit, + "fsurdat_out is specified in both the command line and the config file, pick one", + ): + fsurdat_modifier(parser) + + def test_opt_sections(self): + """ + Test that a simple file with the optional sections works + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_opt_sections.cfg") + outfile = os.path.join( + self._tempdir, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_output_urban.nc", + ) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + os.path.join( + self._testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc" + ), + "-o", + outfile, + ] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + # Read the resultant output file and make sure the fields are changed as expected + fsurdat_out_data = xr.open_dataset(outfile) + zero0d = np.zeros((5, 5)) + one0d = np.ones((5, 5)) + pct_urban = np.array( + [ + [ + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ) + lev2_two = np.empty((2, 3, 5, 5)) + lev2_two[0, :, :, :] = 200.0 + lev2_two[1, :, :, :] = 100.0 + lev2_five = np.empty((5, 3, 5, 5)) + lev2_five[0, :, :, :] = 1.0 + lev2_five[1, :, :, :] = 2.0 + lev2_five[2, :, :, :] = 3.0 + lev2_five[3, :, :, :] = 4.0 + lev2_five[4, :, :, :] = 5.0 + lev1 = np.array( + [ + [ + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + ], + [ + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + ], + [ + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + ], + ] + ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_NATVEG, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_CROP, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_WETLAND, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_GLACIER, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_URBAN, pct_urban) + np.testing.assert_array_equal(fsurdat_out_data.LAKEDEPTH, one0d * 200.0) + np.testing.assert_array_equal(fsurdat_out_data.T_BUILDING_MIN, lev1) + np.testing.assert_array_equal(fsurdat_out_data.ALB_ROOF_DIR, lev2_two) + np.testing.assert_array_equal(fsurdat_out_data.TK_ROOF, lev2_five) + + def test_1x1_mexicocity(self): + """ + Test that the mexicocity file is handled correctly + """ + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_1x1mexicocity.cfg" + ) + expectfile = os.path.join( + self._testinputs_path, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc", + ) + outfile = os.path.join( + self._tempdir, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc", + ) + infile = os.path.join( + self._testinputs_path, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", + ) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + infile, + "-o", + outfile, + ] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + + # Read the resultant output file and make sure the fields are changed as expected + fsurdat_out_data = xr.open_dataset(outfile) + fsurdat_inp_data = xr.open_dataset(infile) + fsurdat_exp_data = xr.open_dataset(expectfile) + + self.assertFalse(fsurdat_out_data.equals(fsurdat_inp_data)) + # assert that fsurdat_out equals fsurdat_out_baseline + self.assertTrue(fsurdat_out_data.equals(fsurdat_exp_data)) + + def test_cfg_file_DNE_fail(self): + """ + Test that if the config file does not exist that it gracefully fails + """ + self._cfg_file_path = os.path.join(self._tempdir, "FILE_DOES_NOT_EXIST.cfg") + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + with self.assertRaisesRegex(SystemExit, "Config file does NOT exist"): + fsurdat_modifier_arg_process() + + def test_input_fsurdat_DNE_fail(self): + """ + Test that if the input fsurdat file does not exist that it gracefully fails + """ + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) + sys.argv = ["fsurdat_modifier", self._cfg_file_path, "-i", "FILE_DOES_NOT_EXIST.nc"] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex(SystemExit, "Input fsurdat_in file does NOT exist"): + fsurdat_modifier(parser) + + def test_output_fsurdat_EXISTS_fail(self): + """ + Test that if the output fsurdat file does exist that it gracefully fails + without --overwrite option + """ + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + self._cfg_file_path, + "-o", + self._cfg_file_path, + ] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex(SystemExit, "Output file already exists"): + fsurdat_modifier(parser) + + def test_cfg_file_empty_fail(self): + """ + Test that if the config file is empty it gracefully fails + """ + self._cfg_file_path = os.path.join(self._tempdir, "EMPTY_FILE.cfg") + fil = open(self._cfg_file_path, "w") + fil.close() + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): + fsurdat_modifier(parser) + def test_minimalInfo(self): """ This test specifies a minimal amount of information @@ -62,7 +324,9 @@ def test_minimalInfo(self): self._create_config_file_minimal() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below @@ -80,7 +344,9 @@ def test_crop(self): self._create_config_file_crop() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below @@ -105,7 +371,14 @@ def test_allInfo(self): self._create_config_file_complete() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "--allow_ideal_and_include_non_veg", + "--allow_dom_pft_and_idealized", + ] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py new file mode 100644 index 0000000000..c9ee23bac3 --- /dev/null +++ b/python/ctsm/test/test_unit_config_utils.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +"""Unit tests for config_utils +""" + +import unittest + +from configparser import ConfigParser + +from ctsm import unit_testing +from ctsm.config_utils import lon_range_0_to_360, get_config_value_or_array + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + +# pylint: disable=protected-access + + +class TestConfigUtils(unittest.TestCase): + """Tests of config_utils""" + + # Allow these to be set outside of the __init__ method + # pylint: disable=attribute-defined-outside-init + def setUp(self): + """Setup for testing""" + self.config = ConfigParser() + self.section = "main" + self.file_path = "path_to_file" + self.config[self.section] = {} + + def test_negative_lon(self): + """Test lon_range_0_to_360 for a negative longitude""" + lon = -180.0 + lon_new = lon_range_0_to_360(lon) + self.assertEqual(lon_new, 180.0, "lon not as expected") + + def test_negative2_lon(self): + """Test lon_range_0_to_360 for a negative longitude""" + lon = -5.0 + lon_new = lon_range_0_to_360(lon) + self.assertEqual(lon_new, 355.0, "lon not as expected") + + def test_regular_lon(self): + """Test lon_range_0_to_360 for a regular longitude""" + lon = 22.567 + lon_new = lon_range_0_to_360(lon) + self.assertEqual(lon_new, lon, "lon not as expected") + + def test_lon_out_of_range(self): + """Test lon_range_0_to_360 for longitude out of range""" + lon = 361.0 + with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): + lon_range_0_to_360(lon) + + def test_lon_out_of_range_negative(self): + """Test lon_range_0_to_360 for longitude out of range""" + lon = -181.0 + with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): + lon_range_0_to_360(lon) + + def test_config_value_or_array_single_value(self): + """Simple test of get_config_value_or_array""" + item = "single_value_thing" + # Test on a string, float and integer + self.config.set(self.section, item, "one-thing") + value = get_config_value_or_array(self.config, self.section, item) + self.assertEqual(value, "one-thing", "Value as expected") + self.config.set(self.section, item, "100.") + value = get_config_value_or_array(self.config, self.section, item) + self.assertEqual(value, "100.", "Value as expected") + self.config.set(self.section, item, "100") + value = get_config_value_or_array(self.config, self.section, item) + self.assertEqual(value, "100", "Value as expected") + # Run over again, with an explicit conversion + self.config.set(self.section, item, "one-thing") + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=str) + self.assertEqual(value, "one-thing", "Value as expected") + self.config.set(self.section, item, "100.") + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=float) + self.assertEqual(value, 100.0, "Value as expected") + self.config.set(self.section, item, "100") + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=int) + self.assertEqual(value, 100, "Value as expected") + + def test_config_value_or_array_for_list(self): + """Simple test of get_config_value_or_array for a list""" + item = "three_things" + # Test on a string, float and integer + mystr = "one two three" + mystrlist = ["one", "two", "three"] + myfloat = "1. 2. 3." + myfloatlist = [1.0, 2.0, 3.0] + myint = "1 2 3" + myintlist = [1, 2, 3] + self.config.set(self.section, item, mystr) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=str) + self.assertEqual(value, mystrlist, "List as expected") + self.assertEqual(len(value), 3, "List size as expected") + self.config.set(self.section, item, myfloat) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=float) + self.assertEqual(value, myfloatlist, "Value as expected") + self.assertEqual(len(value), 3, "List size as expected") + self.config.set(self.section, item, myint) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=int) + self.assertEqual(value, myintlist, "Value as expected") + self.assertEqual(len(value), 3, "List size as expected") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py new file mode 100755 index 0000000000..0c7a244584 --- /dev/null +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 + +""" +Unit tests for fsurdat_modifier subroutines: +""" + +import unittest +import os +import sys +import shutil + +import tempfile +from configparser import ConfigParser +import xarray as xr + +from ctsm import unit_testing +from ctsm.path_utils import path_to_ctsm_root +from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_subgrid +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_option_control +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_var_list +from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section +from ctsm.modify_input_files.fsurdat_modifier import check_no_varlist_section +from ctsm.modify_input_files.modify_fsurdat import ModifyFsurdat + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + +# pylint: disable=protected-access + + +# Allow as many public methods as needed... +# pylint: disable=too-many-public-methods +# Allow all the instance attributes that we need +# pylint: disable=too-many-instance-attributes +class TestFSurdatModifier(unittest.TestCase): + """Tests the fsurdat_modifier subroutines""" + + def setUp(self): + """Setup for trying out the methods""" + testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._cfg_file_path = os.path.join(testinputs_path, "modify_fsurdat_opt_sections.cfg") + self._testinputs_path = testinputs_path + self._fsurdat_in = os.path.join( + testinputs_path, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", + ) + self._tempdir = tempfile.mkdtemp() + self._fsurdat_in = os.path.join( + testinputs_path, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", + ) + self._fsurdat_out = os.path.join(self._tempdir, "fsurdat_out.nc") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + self._fsurdat_in, + "-o", + self._fsurdat_out, + ] + parser = fsurdat_modifier_arg_process() + self.cfg_path = str(parser.cfg_path) + self.config = ConfigParser() + self.config.read(self.cfg_path) + my_data = xr.open_dataset(self._fsurdat_in) + self.modify_fsurdat = ModifyFsurdat( + my_data=my_data, + lon_1=0.0, + lon_2=360.0, + lat_1=90.0, + lat_2=90.0, + landmask_file=None, + lat_dimname=None, + lon_dimname=None, + ) + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_dom_pft_and_idealized_fails(self): + """test a that dom_pft and idealized fails gracefully""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "dom_pft", "1") + with self.assertRaisesRegex( + SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" + ): + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + + def test_subgrid_and_idealized_fails(self): + """test that dom_pft and idealized fails gracefully""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "include_nonveg", "False") + self.config.set(section, "process_subgrid_section", "True") + self.config.set(section, "dom_pft", "UNSET") + with self.assertRaisesRegex( + SystemExit, + "idealized AND process_subgrid_section can NOT both be on, pick one or the other", + ): + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + + def test_optional_only_true_and_false(self): + """test that optional settings can only be true or false""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "dom_pft", "1") + varlist = ( + "idealized", + "include_nonveg", + "process_subgrid_section", + "process_var_list_section", + ) + for var in varlist: + self.config.set(section, var, "True") + self.config.set(section, "idealized", "False") + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + for var in varlist: + self.config.set(section, var, "False") + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + self.config.set(section, "dom_pft", "UNSET") + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + var = "idealized" + self.config.set(section, var, "Thing") + with self.assertRaisesRegex( + SystemExit, "Non-boolean value found for .cfg file variable: " + var + ): + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + + def test_include_nonveg_and_idealized_fails(self): + """test a simple read of subgrid""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "include_nonveg", "True") + with self.assertRaisesRegex( + SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" + ): + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + + def test_read_subgrid(self): + """test a simple read of subgrid""" + read_cfg_subgrid(self.config, self.cfg_path) + + def test_read_subgrid_allglacier(self): + """test a read of subgrid that's for all glacier""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0. 0. 0.") + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "100.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_cfg_subgrid(self.config, self.cfg_path) + + def test_read_subgrid_allspecial(self): + """test a read of subgrid that's all special landunits""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0. 0. 0.") + self.config.set(section, "pct_lake", "25.") + self.config.set(section, "pct_wetland", "35.") + self.config.set(section, "pct_glacier", "40.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_cfg_subgrid(self.config, self.cfg_path) + + def test_read_subgrid_allurban(self): + """test a read of subgrid that's all urban""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100.0 0.0 0.0") + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "0.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_cfg_subgrid(self.config, self.cfg_path) + + def test_read_var_list(self): + """test a simple read of var_list""" + read_cfg_var_list(self.config, idealized=True) + + def test_subgrid_outofrange(self): + """test a read of subgrid that's out of range""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "101. 0. 0.") + with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_pct_urban_toosmall(self): + """test a read of subgrid for PCT_URBAN that's an array too small""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100. 0.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_pct_urban_toobig(self): + """test a read of subgrid for PCT_URBAN that's an array too big""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100. 0. 0. 0.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_pct_urban_singlevalue(self): + """test a read of subgrid for PCT_URBAN that's a single value""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_notsumtohundred(self): + """test a read of subgrid that's doesn't sum to a hundred""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0. 0. 0.") + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "0.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + with self.assertRaisesRegex( + SystemExit, "PCT fractions in subgrid section do NOT sum to a hundred as they should" + ): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_badvar(self): + """test a read of subgrid for a variable thats not in the list""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "badvariable", "100.") + with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_varlist_varinidealized(self): + """test a read of varlist for a variable thats in the idealized list, + when idealized is on""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + section = "modify_fsurdat_variable_list" + self.config.set(section, "PCT_SAND", "100.") + with self.assertRaisesRegex( + SystemExit, + "is a special variable handled in the idealized section." + + " This should NOT be handled in the variiable list section. Special idealized vars =", + ): + read_cfg_var_list(self.config, idealized=True) + + def test_varlist_varinsubgrid(self): + """test a read of varlist for a variable thats in the subgrid list""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") + section = "modify_fsurdat_variable_list" + self.config.set(section, "PCT_GLACIER", "100.") + with self.assertRaisesRegex( + SystemExit, + "is a variable handled in the subgrid section." + + " This should NOT be handled in the variiable list section. Subgrid vars =", + ): + read_cfg_var_list(self.config, idealized=False) + + def test_varlist_monthlyvar(self): + """test a read of varlist for a variable thats one of the monthly + variables handled in the dom_pft section""" + section = "modify_fsurdat_variable_list" + self.config.set(section, "MONTHLY_LAI", "100.") + with self.assertRaisesRegex( + SystemExit, + "is a variable handled as part of the dom_pft handling." + + " This should NOT be handled in the variiable list section." + + " Monthly vars handled this way =", + ): + read_cfg_var_list(self.config, idealized=False) + + def test_subgrid_remove(self): + """test a read of subgrid when it's section has been removed""" + section = "modify_fsurdat_subgrid_fractions" + self.config.remove_section(section) + with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): + read_cfg_subgrid(self.config, self.cfg_path) + + def test_subgrid_not_thereifoff(self): + """test that a graceful error happens if subgrid section is off, + but it appears in the file""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "process_subgrid_section", "False") + with self.assertRaisesRegex(SystemExit, "Config file does have a section"): + check_no_subgrid_section(self.config) + + def test_varlist_not_thereifoff(self): + """test that a graceful error happens if varlist section is off, + but it appears in the file""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "process_var_list_section", "False") + with self.assertRaisesRegex(SystemExit, "Config file does have a section"): + check_no_varlist_section(self.config) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index e53175b9b7..a075035b73 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -19,87 +19,102 @@ # pylint: disable=protected-access +## Too many instant variables as part of the class (too many self. in the SetUp) +# pylint: disable=too-many-instance-attributes + class TestModifyFsurdat(unittest.TestCase): """Tests the setvar_lev functions and the - _get_rectangle function + _get_rectangle, check_varlist, and set_varlist methods """ - def test_setvarLev(self): - """ - Tests that setvar_lev0, setvar_lev1, and setvar_lev2 update values of - variables within a rectangle defined by user-specified - lon_1, lon_2, lat_1, lat_2 - """ + def setUp(self): # get longxy, latixy that would normally come from an fsurdat file # self._get_longxy_latixy will convert -180 to 180 to 0-360 longitudes # get cols, rows also - min_lon = 2 # expects min_lon < max_lon - min_lat = 3 # expects min_lat < max_lat - longxy, latixy, cols, rows = self._get_longxy_latixy( - _min_lon=min_lon, _max_lon=10, _min_lat=min_lat, _max_lat=12 + self.min_lon = 2 # expects min_lon < max_lon + self.min_lat = 3 # expects min_lat < max_lat + longxy, latixy, self.cols, self.rows = self._get_longxy_latixy( + _min_lon=self.min_lon, _max_lon=10, _min_lat=self.min_lat, _max_lat=12 ) # get not_rectangle from user-defined lon_1, lon_2, lat_1, lat_2 - lon_1 = 3 - lon_2 = 5 # lon_1 < lon_2 - lat_1 = 5 - lat_2 = 7 # lat_1 < lat_2 + self.lon_1 = 3 + self.lon_2 = 5 # lon_1 < lon_2 + self.lat_1 = 5 + self.lat_2 = 7 # lat_1 < lat_2 # create xarray dataset containing lev0, lev1, and lev2 variables; # the fsurdat_modify tool reads variables like this from fsurdat file - var_1d = np.arange(cols) - var_lev2 = var_1d * np.ones((rows, cols, rows, cols)) - var_lev1 = var_1d * np.ones((cols, rows, cols)) + var_1d = np.arange(self.cols) + var_lev0 = var_1d * np.ones((self.rows, self.cols)) + var_lev1 = var_1d * np.ones((self.cols, self.rows, self.cols)) + var_lev2 = var_1d * np.ones((self.rows, self.cols, self.rows, self.cols)) + var_lev3 = var_1d * np.ones((self.cols, self.rows, self.cols, self.rows, self.cols)) my_data = xr.Dataset( data_vars=dict( LONGXY=(["x", "y"], longxy), # use LONGXY as var_lev0 LATIXY=(["x", "y"], latixy), # __init__ expects LONGXY, LATIXY + urbdens=(["numurbl"], var_1d), # numurbl needs to be dimension + var_lev0=(["x", "y"], var_lev0), var_lev1=(["w", "x", "y"], var_lev1), var_lev2=(["v", "w", "x", "y"], var_lev2), + var_lev3=(["z", "v", "w", "x", "y"], var_lev3), + VAR_LEV0_UPPERCASE=(["x", "y"], var_lev0), + VAR_LEV1_UPPERCASE=(["w", "x", "y"], var_lev1), + VAR_LEV2_UPPERCASE=(["v", "w", "x", "y"], var_lev2), + VAR_LEV3_UPPERCASE=(["z", "v", "w", "x", "y"], var_lev3), ) ) # create ModifyFsurdat object - modify_fsurdat = ModifyFsurdat( + self.modify_fsurdat = ModifyFsurdat( my_data=my_data, - lon_1=lon_1, - lon_2=lon_2, - lat_1=lat_1, - lat_2=lat_2, + lon_1=self.lon_1, + lon_2=self.lon_2, + lat_1=self.lat_1, + lat_2=self.lat_2, landmask_file=None, lat_dimname=None, lon_dimname=None, ) + def test_setvarLev(self): + """ + Tests that setvar_lev0, setvar_lev1, and setvar_lev2 update values of + variables within a rectangle defined by user-specified + lon_1, lon_2, lat_1, lat_2 + """ + # initialize and then modify the comparison matrices - comp_lev0 = modify_fsurdat.file.LONGXY - comp_lev1 = modify_fsurdat.file.var_lev1 - comp_lev2 = modify_fsurdat.file.var_lev2 + comp_lev0 = self.modify_fsurdat.file.LONGXY + comp_lev1 = self.modify_fsurdat.file.var_lev1 + comp_lev2 = self.modify_fsurdat.file.var_lev2 val_for_rectangle = 1.5 comp_lev0[ - lat_1 - min_lat : lat_2 - min_lat + 1, lon_1 - min_lon : lon_2 - min_lon + 1 + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle comp_lev1[ ..., - lat_1 - min_lat : lat_2 - min_lat + 1, - lon_1 - min_lon : lon_2 - min_lon + 1, + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle comp_lev2[ ..., - lat_1 - min_lat : lat_2 - min_lat + 1, - lon_1 - min_lon : lon_2 - min_lon + 1, + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle # test setvar - modify_fsurdat.setvar_lev0("LONGXY", val_for_rectangle) - np.testing.assert_array_equal(modify_fsurdat.file.LONGXY, comp_lev0) + self.modify_fsurdat.setvar_lev0("LONGXY", val_for_rectangle) + np.testing.assert_array_equal(self.modify_fsurdat.file.LONGXY, comp_lev0) - modify_fsurdat.setvar_lev1("var_lev1", val_for_rectangle, cols - 1) - np.testing.assert_array_equal(modify_fsurdat.file.var_lev1, comp_lev1) + self.modify_fsurdat.setvar_lev1("var_lev1", val_for_rectangle, self.cols - 1) + np.testing.assert_array_equal(self.modify_fsurdat.file.var_lev1, comp_lev1) - modify_fsurdat.setvar_lev2("var_lev2", val_for_rectangle, cols - 1, rows - 1) - np.testing.assert_array_equal(modify_fsurdat.file.var_lev2, comp_lev2) + self.modify_fsurdat.setvar_lev2("var_lev2", val_for_rectangle, self.cols - 1, self.rows - 1) + np.testing.assert_array_equal(self.modify_fsurdat.file.var_lev2, comp_lev2) def test_getNotRectangle_lon1leLon2Lat1leLat2(self): """ @@ -359,6 +374,85 @@ def test_getNotRectangle_latsOutOfBounds(self): latixy=latixy, ) + def test_check_varlist_lists(self): + """Test the check_varlist method for list for dimensions that works""" + lev1list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + lev2list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + settings = {"var_lev1": lev1list, "var_lev2": lev2list} + settings_new = self.modify_fsurdat.check_varlist(settings) + self.assertEqual( + settings_new, settings, "list of variable settings not identical as expected" + ) + + def test_check_varlist_lists_wrongsizes(self): + """Test the check_varlist method for lists to gracefully fail when the sizes are wrong""" + lev1list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + settings = {"var_lev1": lev1list} + with self.assertRaisesRegex( + SystemExit, "Variable var_lev1 is of the wrong size. It should be" + ): + self.modify_fsurdat.check_varlist(settings) + + def test_get_numurb_dens(self): + """Check that get num urban density types is correct""" + self.assertEqual( + self.modify_fsurdat.get_urb_dens(), + 9, + "Default number of urban density types is correct", + ) + + def test_check_varlist_uppercase(self): + """Test the check_varlist method for all the dimensions that + works with allowuppercase option""" + vallist = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + vallist2 = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + expected = { + "VAR_LEV0_UPPERCASE": 100.0, + "VAR_LEV1_UPPERCASE": vallist, + "VAR_LEV2_UPPERCASE": vallist2, + } + settings = { + "var_lev0_uppercase": 100.0, + "var_lev1_uppercase": vallist, + "var_lev2_uppercase": vallist2, + } + settings_new = self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + self.assertEqual( + expected, + settings_new, + "list of variable settings not converted to uppercase as expected", + ) + + def test_check_varlist_badvar(self): + """Test the check_varlist method for a variable not on the file""" + settings = {"badvar": 100.0} + with self.assertRaisesRegex(SystemExit, "Variable badvar is NOT in the file"): + self.modify_fsurdat.check_varlist(settings) + + def test_check_varlist_badvar_uppercase(self): + """Test the check_varlist method for a variable not on the file with allow uppercase""" + settings = {"badvar": 100.0} + with self.assertRaisesRegex(SystemExit, "Variable BADVAR is NOT in the file"): + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + + def test_set_varlist_toohighdim(self): + """Test the set_varlist method for a variable of too high a dimension""" + settings = {"var_lev3": 100.0} + with self.assertRaisesRegex( + SystemExit, "Variable var_lev3 is a higher dimension than currently allowed" + ): + self.modify_fsurdat.set_varlist(settings) + + def test_set_varlist_toohighdim_uppercase(self): + """Test the set_varlist method for a variable of too high a dimension in uppercase""" + settings = {"var_lev3_uppercase": 100.0} + with self.assertRaisesRegex( + SystemExit, + "For higher dimensional vars, the variable needs to be expressed as a " + + "list of values of the dimension size = 9 for variable=VAR_LEV3_UPPERCASE", + ): + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ Return longxy, latixy, cols, rows diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index aa4c412bbb..fd9aef631d 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -8,6 +8,7 @@ import unittest import configparser +import argparse import os import sys @@ -17,7 +18,7 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.subset_data import get_parser, setup_files +from ctsm.subset_data import get_parser, setup_files, check_args from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name @@ -29,10 +30,10 @@ class TestSubsetData(unittest.TestCase): """ def setUp(self): - sys.argv = ["subset_data", "point"] - DEFAULTS_FILE = "default_data.cfg" - parser = get_parser() - self.args = parser.parse_args() + sys.argv = ["subset_data", "point", "--create-surface"] + DEFAULTS_FILE = os.path.join(os.getcwd(), "ctsm/test/testinputs/default_data.cfg") + self.parser = get_parser() + self.args = self.parser.parse_args() self.cesmroot = path_to_ctsm_root() self.defaults = configparser.ConfigParser() self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) @@ -41,20 +42,115 @@ def test_inputdata_setup_files_basic(self): """ Test """ - setup_files(self.args, self.defaults, self.cesmroot) + check_args(self.args) + files = setup_files(self.args, self.defaults, self.cesmroot) + self.assertEqual( + files["fsurf_in"], + "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "fsurf_in filename not whats expected", + ) + self.assertEqual( + files["fsurf_out"], + None, + "fsurf_out filename not whats expected", + ) + self.assertEqual( + files["main_dir"], + "/glade/p/cesmdata/inputdata", + "main_dir directory not whats expected", + ) def test_inputdata_setup_files_inputdata_dne(self): """ Test that inputdata directory does not exist """ + check_args(self.args) self.defaults.set("main", "clmforcingindir", "/zztop") with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) + def test_check_args_nooutput(self): + """ + Test that check args aborts when no-output is asked for + """ + sys.argv = ["subset_data", "point"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex(argparse.ArgumentError, "Must supply one of"): + check_args(self.args) + + def test_check_args_notype(self): + """ + Test that check args aborts when no type is asked for + """ + sys.argv = ["subset_data"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex(argparse.ArgumentError, "Must supply a positional argument:"): + check_args(self.args) + + def test_check_args_badconfig(self): + """ + Test that check args aborts when a config file is entered that doesn't exist + """ + sys.argv = ["subset_data", "point", "--create-surface", "--cfg-file", "zztop"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex( + argparse.ArgumentError, "Entered default config file does not exist" + ): + check_args(self.args) + + def test_check_args_outsurfdat_provided(self): + """ + Test that check args allows an output surface dataset to be specified + when create-surface is on + """ + sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "outputsurface.nc"] + self.args = self.parser.parse_args() + check_args(self.args) + files = setup_files(self.args, self.defaults, self.cesmroot) + self.assertEqual( + files["fsurf_out"], + "outputsurface.nc", + "fsurf_out filename not whats expected", + ) + + def test_check_args_outsurfdat_fails_without_create_surface(self): + """ + Test that check args does not allow an output surface dataset to be specified + when create-surface is not on + """ + sys.argv = ["subset_data", "point", "--create-landuse", "--out-surface", "outputsurface.nc"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex( + argparse.ArgumentError, + "out-surface option is given without the --create-surface option", + ): + check_args(self.args) + + def test_check_args_outsurfdat_fails_without_overwrite(self): + """ + Test that check args does not allow an output surface dataset to be specified + for an existing dataset without the overwrite option + """ + outfile = os.path.join( + os.getcwd(), + "ctsm/test/testinputs/", + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", + ) + self.assertTrue(os.path.exists(outfile), str(outfile) + " outfile should exist") + + sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", outfile] + self.args = self.parser.parse_args() + with self.assertRaisesRegex( + argparse.ArgumentError, + "out-surface filename exists and the overwrite option was not also selected", + ): + check_args(self.args) + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad """ + check_args(self.args) self.args.inputdatadir = "/zztop" with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) diff --git a/python/ctsm/test/testinputs/default_data.cfg b/python/ctsm/test/testinputs/default_data.cfg new file mode 100644 index 0000000000..7e841dca54 --- /dev/null +++ b/python/ctsm/test/testinputs/default_data.cfg @@ -0,0 +1,28 @@ +[main] +clmforcingindir = /glade/p/cesmdata/inputdata + +[datm_gswp3] +dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +domain = domain.lnd.360x720_gswp3.0v1.c170606.nc +solardir = Solar +precdir = Precip +tpqwdir = TPHWL +solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. +prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. +tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. +solarname = CLMGSWP3v1.Solar +precname = CLMGSWP3v1.Precip +tpqwname = CLMGSWP3v1.TPQW + +[surfdat] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +surfdat_16pft = surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +surfdat_78pft = surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_c190214.nc + +[landuse] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +landuse_16pft = landuse.timeseries_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc +landuse_78pft = landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_c190214.nc + +[domain] +file = share/domains/domain.lnd.fv0.9x1.25_gx1v7.151020.nc diff --git a/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg new file mode 100644 index 0000000000..a4118a3255 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg @@ -0,0 +1,84 @@ +[modify_fsurdat_basic_options] + +idealized = False +process_subgrid_section = True +process_var_list_section = True +include_nonveg = True + +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area + +# Variables on numurbl which is 3 +CANYON_HWR = 1.18 1.18 1.18 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.9 0.9 0.9 +EM_WALL = 0.85 0.85 0.85 +HT_ROOF = 18.8 18.8 18.8 +THICK_ROOF = 0.185 0.185 0.185 +THICK_WALL = 0.45 0.45 0.45 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 9.4 9.4 9.4 +WTLUNIT_ROOF = 0.55 0.55 0.55 +WTROAD_PERV = 0.04 0.04 0.04 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.2 0.2 +ALB_ROOF_DIF = 0.2 0.2 +ALB_WALL_DIR = 0.25 0.25 +ALB_WALL_DIF = 0.25 0.25 + +# Variabls on nlevurb which is 5 +TK_ROOF = 0.20 0.93 0.93 0.03 0.16 +TK_WALL = 0.88 0.88 0.88 0.88 0.88 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1500000.0 1500000.0 250000.0 870000.0 +CV_WALL = 1540000.0 1540000.0 1540000.0 1540000.0 1540000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg new file mode 100644 index 0000000000..c068c5d851 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -0,0 +1,51 @@ +[modify_fsurdat_basic_options] + +idealized = False +process_subgrid_section = True +process_var_list_section = True +include_nonveg = True + +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# Set to 100% urban for the test +PCT_NATVEG = 0.0 +PCT_CROP = 0.0 +PCT_LAKE = 0.0 +PCT_GLACIER = 0.0 +PCT_WETLAND = 0.0 +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 + + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# Set lake-depth to 200 for the test +LAKEDEPTH = 200.00 + +# Set soem urban multidimensional variables to 200 for the test +# If given as a single value set all to the value, if as list set by dimension and if array as an array +CANYON_HWR = 200.00 150.0 100. +HT_ROOF = 200.0 150.0 100. +T_BUILDING_MIN = 200 150.0 100. +ALB_ROOF_DIR = 200. 100. +TK_ROOF = 1. 2. 3. 4. 5. diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg new file mode 100644 index 0000000000..74c6639899 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg @@ -0,0 +1,29 @@ +[modify_fsurdat_basic_options] + +fsurdat_in = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc +fsurdat_out = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc + +idealized = False +process_subgrid_section = False +process_var_list_section = False +include_nonveg = False + +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg new file mode 100644 index 0000000000..3e5aada24b --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg @@ -0,0 +1,11 @@ +[modify_fsurdat_basic_options] + +idealized = False +process_subgrid_section = False +process_var_list_section = False +include_nonveg = False + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 diff --git a/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc new file mode 100644 index 0000000000..4bf009ebe6 --- /dev/null +++ b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef672e3ab2c237fd6f76afdd1dfa7d01263c01bd94bef2a8f37acfba3a9b6e36 +size 27300 diff --git a/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc new file mode 100644 index 0000000000..80a66c286f --- /dev/null +++ b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfc14c3f7e9efa4f0acc4caca22b616e09e04b2713ab1424d695492ac52327c +size 27772 diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index e531dac39a..0a940df0da 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -4294,6 +4294,7 @@ subroutine Summary_carbonflux(this, & ! root respiration (RR) this%rr_patch(p) = & this%froot_mr_patch(p) + & + this%livecroot_mr_patch(p) + & this%cpool_froot_gr_patch(p) + & this%cpool_livecroot_gr_patch(p) + & this%cpool_deadcroot_gr_patch(p) + & diff --git a/src/biogeophys/CanopyFluxesMod.F90 b/src/biogeophys/CanopyFluxesMod.F90 index 0dc9bbf797..038f8ea636 100644 --- a/src/biogeophys/CanopyFluxesMod.F90 +++ b/src/biogeophys/CanopyFluxesMod.F90 @@ -1329,12 +1329,7 @@ subroutine CanopyFluxes(bounds, num_exposedvegp, filter_exposedvegp, zeta(p) = zldis(p)*vkc*grav*thvstar/(ustar(p)**2*thv(c)) if (zeta(p) >= 0._r8) then !stable - ! remove stability cap when biomass heat storage is active - if(use_biomass_heat_storage) then - zeta(p) = min(100._r8,max(zeta(p),0.01_r8)) - else - zeta(p) = min(zetamax,max(zeta(p),0.01_r8)) - endif + zeta(p) = min(zetamax,max(zeta(p),0.01_r8)) um(p) = max(ur(p),0.1_r8) else !unstable zeta(p) = max(-100._r8,min(zeta(p),-0.01_r8)) diff --git a/test/tools/input_tests_master b/test/tools/input_tests_master index f3e46d50b5..63e4e5173f 100644 --- a/test/tools/input_tests_master +++ b/test/tools/input_tests_master @@ -24,15 +24,6 @@ bli58 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_10x15_crp_1850-200 smi64 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds bli64 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds -smi74 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850-2000^tools__ds -bli74 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850-2000^tools__ds -smi78 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850^tools__ds -bli78 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850^tools__ds -smiT4 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_2000^tools__ds -bliT4 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_2000^tools__ds -smiT2 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100^tools__s -bliT2 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100^tools__s - sm0a1 TSMscript_tools.sh site_and_regional run_neon.py run_neon_OSBS bl0a1 TBLscript_tools.sh site_and_regional run_neon.py run_neon_OSBS @@ -54,5 +45,3 @@ smi#2 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_ne30np4 bli#2 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_ne30np4 smi59 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_if10 bli59 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_if10 -smi79 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_i1x1_brazil -bli79 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_i1x1_brazil diff --git a/test/tools/nl_files/mkmapdata_i1x1_brazil b/test/tools/nl_files/mkmapdata_i1x1_brazil deleted file mode 100644 index bb54d468dc..0000000000 --- a/test/tools/nl_files/mkmapdata_i1x1_brazil +++ /dev/null @@ -1 +0,0 @@ --t regional -r 1x1_brazil --fast diff --git a/test/tools/nl_files/mksrfdt_1x1_brazil_1850 b/test/tools/nl_files/mksrfdt_1x1_brazil_1850 deleted file mode 100644 index 2330bd082e..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_brazil_1850 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_brazil -y 1850-2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 b/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 deleted file mode 100644 index 2330bd082e..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_brazil -y 1850-2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 b/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 deleted file mode 100644 index 03304f81eb..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_numaIA -y 2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 b/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 deleted file mode 100644 index ed83434075..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_numaIA -y 1850-2100 -ssp_rcp SSP5-8.5 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 b/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 deleted file mode 100644 index a446e82fcd..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_vancouverCAN -no-crop -y 2000 -exedir EXEDIR diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index ce501f980a..6a6749d493 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -373,8 +373,11 @@ else fi # Setup conda environement -\$CLM_ROOT/py_env_create conda activate ctsm_py +if [ \$? -ne 0 ]; then + echo "ERROR: Trouble activating the ctsm_py conda environment, be sure it's setup with \$CLM_ROOT/py_env_create, then rerun" + exit 4 +fi ##output files clm_log=\${initdir}/td.\${JOBID}.log diff --git a/test/tools/tests_posttag_hobart_nompi b/test/tools/tests_posttag_hobart_nompi index 9f07863e4d..d3cbccecdf 100644 --- a/test/tools/tests_posttag_hobart_nompi +++ b/test/tools/tests_posttag_hobart_nompi @@ -1,4 +1,3 @@ smc#4 blc#4 smi54 bli54 smi57 bli57 -smiT4 bliT4 diff --git a/test/tools/tests_posttag_izumi_nompi b/test/tools/tests_posttag_izumi_nompi index 62687a7e3d..3e84fb8459 100644 --- a/test/tools/tests_posttag_izumi_nompi +++ b/test/tools/tests_posttag_izumi_nompi @@ -1,3 +1,2 @@ smi54 bli54 smi57 bli57 -smiT4 bliT4 diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression index 1395aebe11..b665409c51 100644 --- a/test/tools/tests_posttag_nompi_regression +++ b/test/tools/tests_posttag_nompi_regression @@ -5,7 +5,3 @@ smi53 bli53 smi54 bli54 smi57 bli57 smi58 bli58 -smi74 bli74 -smi78 bli78 -smiT4 bliT4 -smiT2 bliT2 diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index 19e96594bf..6ce4972915 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -1,4 +1,3 @@ -smi79 bli79 smc#4 blc#4 smg54 blg54 smba1 blba1 @@ -10,6 +9,3 @@ smi64 bli64 smi54 bli54 smi57 bli57 smi58 bli58 -smi74 bli74 -smiT4 bliT4 -smiT2 bliT2 diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index d0c000ba63..c1df608436 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -53,7 +53,29 @@ else endif MKSURFDATA = $(BATCHJOBS) $(PWD)/mksurfdata.pl +SUBSETDATA = $(PWD)/../site_and_regional/subset_data +MODIFYSURF = $(PWD)/../modify_input_files/fsurdat_modifier --overwrite +CDATE = $(shell date +%y%m%d) + +# subset_data options +# +SUBSETDATA_POINT = $(SUBSETDATA) point --silent --overwrite --uniform-snowpack --cap-saturation --crop --outdir . +SUBSETDATA_POINT_ALLLU = $(SUBSETDATA_POINT) --include-nonveg +SUBSETDATA_POINT_URBAN = $(SUBSETDATA_POINT) --include-nonveg + +# Subset data sites... +SUBSETDATA_1X1_BRAZIL := --lat -7 --lon -55 --site 1x1_brazil +SUBSETDATA_1X1_NUMAIA := --lat 40.6878 --lon 267.0228 --site 1x1_numaIA +SUBSETDATA_1X1_SMALL := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA \ + --dompft 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 \ + --pctpft 6.5 1.5 1.6 1.7 1.8 1.9 1.5 1.6 1.7 1.8 1.9 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 +# NOTE: The 1850 smallvilleIA site is constructed to start with 100% natural vegetation, so we can test transition to crops +SUBSETDATA_1X1_SMALL1850 := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA --dompft 13 --pctpft 100 + +SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX --out-surface surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc +SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN --out-surface surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc +SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --out-surface surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -91,14 +113,24 @@ CROP = \ crop-global-historical \ crop-global-transient \ crop-global-future - all : standard tropics crop urban landuse-timeseries +all-subset : \ + 1x1_brazil-tropics-present \ + crop-tropics-historical \ + crop-tropics-transient \ + crop-numa-present \ + crop-numa-historical \ + crop-smallville \ + crop-smallville-historical \ + urban-present urban-alpha + DEBUG: @echo "HOST := $(HOST)" @echo "PROJECT := $(PROJECT)" @echo "BATCHJOBS := $(BATCHJOBS)" @echo "BACKGROUND := $(BACKGROUND)" + # # standard # @@ -119,14 +151,18 @@ global-present-nldas : FORCE # tropics : $(TROPICS) -crop-tropics-present : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon,1x1_brazil $(BACKGROUND) +crop-tropics-present : brazil-tropics-present + $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon $(BACKGROUND) + +1x1_brazil-tropics-present : FORCE + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) + crop-tropics-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --cfg-file default_data_1850.cfg crop-tropics-transient : FORCE - $(MKSURFDATA) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA_POINT_ALLLU) --create-landuse $(SUBSETDATA_1X1_BRAZIL) # # crop @@ -143,16 +179,13 @@ crop-global-present-f05 : FORCE $(MKSURFDATA) -glc_nec 10 -y 1850,2000 -res 0.47x0.63 $(BACKGROUND) crop-numa-present : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) crop-numa-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --cfg-file default_data_1850.cfg crop-smallville : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ - -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ - -pft_frc 6.5,1.5,1.6,1.7,1.8,1.9,1.5,1.6,1.7,1.8,1.9,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5 \ - $(BACKGROUND) + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL) crop-global-present-ne16np4 : FORCE $(MKSURFDATA) -glc_nec 10 -y 2000 -res ne16np4 $(BACKGROUND) @@ -165,7 +198,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 $(BACKGROUND) + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --cfg-file default_data_1850.cfg # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE @@ -229,20 +262,31 @@ crop-global-SSP5-8.5 : FORCE # urban : urban-present urban-alpha -urban-present : FORCE - $(MKSURFDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX $(BACKGROUND) +urban-present : mexicocity vancouver + +mexicocity : FORCE + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) + $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc + +vancouver : FORCE + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) + $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE - $(MKSURFDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check $(BACKGROUND) - + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) + $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc # # landuse timeseries # landuse-timeseries : landuse-timeseries-smallville +# NOTE: TODO: This needs to be chagned to use subset_data when transient configurations are resolved (see Issue #1673 landuse-timeseries-smallville : FORCE $(MKSURFDATA) -no_surfdata -glc_nec 10 -y 1850-1855 -r 1x1_smallvilleIA \ -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ diff --git a/tools/mksurfdata_map/default_data_1850.cfg b/tools/mksurfdata_map/default_data_1850.cfg new file mode 100644 index 0000000000..311aeef13d --- /dev/null +++ b/tools/mksurfdata_map/default_data_1850.cfg @@ -0,0 +1,29 @@ +[main] +clmforcingindir = /glade/p/cesmdata/inputdata + +[datm_gswp3] +dir = /glade/p/cgd/tss/CTSM_datm_forcing_data/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +domain = domain.lnd.360x720_gswp3.0v1.c170606.nc +solardir = Solar +precdir = Precip +tpqwdir = TPHWL +solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. +prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. +tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. +solarname = CLMGSWP3v1.Solar +precname = CLMGSWP3v1.Precip +tpqwname = CLMGSWP3v1.TPQW + +[surfdat] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +surfdat_16pft = surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc +surfdat_78pft = surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr1850_c190214.nc + +[landuse] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +landuse_16pft = landuse.timeseries_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc +landuse_78pft = landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_c190214.nc + +[domain] +file = share/domains/domain.lnd.fv0.9x1.25_gx1v7.151020.nc + diff --git a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg new file mode 100644 index 0000000000..5e9de7968b --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg @@ -0,0 +1,134 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. +idealized = False + +# subgrid section to set the PCT_* variables +process_subgrid_section = True +# Variable list section to set specific variable names +process_var_list_section = True + +# Boundaries of user-defined rectangle to apply changes (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +# Include other land units besides vegetated +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area or other things that are above. + +# Variables on numurbl which is 3 +CANYON_HWR = 1.18 1.18 1.18 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.9 0.9 0.9 +EM_WALL = 0.85 0.85 0.85 +HT_ROOF = 18.8 18.8 18.8 +THICK_ROOF = 0.185 0.185 0.185 +THICK_WALL = 0.45 0.45 0.45 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 9.4 9.4 9.4 +WTLUNIT_ROOF = 0.55 0.55 0.55 +WTROAD_PERV = 0.04 0.04 0.04 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.2 0.2 +ALB_ROOF_DIF = 0.2 0.2 +ALB_WALL_DIR = 0.25 0.25 +ALB_WALL_DIF = 0.25 0.25 + +# Variabls on nlevurb which is 5 +TK_ROOF = 0.20 0.93 0.93 0.03 0.16 +TK_WALL = 0.88 0.88 0.88 0.88 0.88 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1500000.0 1500000.0 250000.0 870000.0 +CV_WALL = 1540000.0 1540000.0 1540000.0 1540000.0 1540000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg new file mode 100644 index 0000000000..38b8ce6372 --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg @@ -0,0 +1,134 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. +idealized = False + +# subgrid section to set the PCT_* variables +process_subgrid_section = True +# Variable list section to set specific variable names +process_var_list_section = True + +# Boundaries of user-defined rectangle to apply changes (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +# Include other land units besides vegetated +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area or other things that are above. + +# Variables on numurbl which is 3 +CANYON_HWR = 0.42 0.42 0.42 +EM_IMPROAD = 0.973 0.973 0.973 +EM_PERROAD = 0.973 0.973 0.973 +EM_ROOF = 0.973 0.973 0.973 +EM_WALL = 0.973 0.973 0.973 +HT_ROOF = 6.4 6.4 6.4 +THICK_ROOF = 0.1141 0.1141 0.1141 +THICK_WALL = 0.1489 0.1489 0.1489 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 3.2 3.2 3.2 +WTLUNIT_ROOF = 0.445 0.445 0.445 +WTROAD_PERV = 0.68 0.68 0.68 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 4 4 4 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.15 0.15 +ALB_IMPROAD_DIF = 0.15 0.15 +ALB_PERROAD_DIR = 0.15 0.15 +ALB_PERROAD_DIF = 0.15 0.15 +ALB_ROOF_DIR = 0.21 0.21 +ALB_ROOF_DIF = 0.21 0.21 +ALB_WALL_DIR = 0.21 0.21 +ALB_WALL_DIF = 0.21 0.21 + +# Variabls on nlevurb which is 5 +TK_ROOF = 6.530 0.025 0.230 0.160 0.00 +TK_WALL = 0.610 0.430 0.024 0.160 0.00 +TK_IMPROAD = 1.170 0.300 0.300 0.420 0.00 +CV_ROOF = 2070000.0 7100.0 1500000.0 670000.0 0.0 +CV_WALL = 1250000.0 1400000.0 1300.0 670000.0 0.0 +CV_IMPROAD = 1140000.0 1050000.0 1050000.0 1290000.0 0.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg new file mode 100644 index 0000000000..abc5df16b1 --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg @@ -0,0 +1,134 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. +idealized = False + +# subgrid section to set the PCT_* variables +process_subgrid_section = True +# Variable list section to set specific variable names +process_var_list_section = True + +# Boundaries of user-defined rectangle to apply changes (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +# Include other land units besides vegetated +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area or other things that are above. + +# Variables on numurbl which is 3 +CANYON_HWR = 0.39 0.39 0.39 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.92 0.92 0.92 +EM_WALL = 0.90 0.90 0.90 +HT_ROOF = 5.8 5.8 5.8 +THICK_ROOF = 0.070 0.070 0.070 +THICK_WALL = 0.20 0.20 0.20 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 2.9 2.9 2.9 +WTLUNIT_ROOF = 0.51 0.51 0.51 +WTROAD_PERV = 0.11 0.11 0.11 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.12 0.12 +ALB_ROOF_DIF = 0.12 0.12 +ALB_WALL_DIR = 0.50 0.50 +ALB_WALL_DIF = 0.50 0.50 + +# Variabls on nlevurb which is 5 +TK_ROOF = 1.40 1.40 1.40 0.03 1.51 +TK_WALL = 1.51 1.51 0.67 0.67 1.51 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1760000.0 1760000.0 40000.0 2210000.0 +CV_WALL = 2110000.0 2110000.0 1000000.0 1000000.0 2110000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/tools/modify_input_files/modify_fsurdat_template.cfg b/tools/modify_input_files/modify_fsurdat_template.cfg index 0694174666..3661784521 100644 --- a/tools/modify_input_files/modify_fsurdat_template.cfg +++ b/tools/modify_input_files/modify_fsurdat_template.cfg @@ -1,4 +1,4 @@ -[modify_input] +[modify_fsurdat_basic_options] # ------------------------------------------------------------------------ # .cfg file with inputs for fsurdat_modifier. @@ -37,7 +37,14 @@ fsurdat_out = FILL_THIS_IN # ORGANIC = 0 idealized = False -# Boundaries of user-defined rectangle (float) + +# Process the optional section that handles modifying subgrid fractions +process_subgrid_section = False + +# Process the optional section that handles modifying an arbitrary list of variables +process_var_list_section = False + +# Boundaries of user-defined rectangle to apply changes (float) # If lat_1 > lat_2, the code creates two rectangles, one in the north and # one in the south. # If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian.