-
Notifications
You must be signed in to change notification settings - Fork 378
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #404 from billsacks/two_part_system_tests_clone
Infrastructure for clone-based two-part system tests This adds infrastructure for doing two-part system tests (i.e., our typical system tests, where you do two runs and compare them) via clones. From discussion on a recent CIME telecon, there was general agreement that we want to migrate our system tests to this style where possible: It's easier to write, understand and maintain tests set up like this, where there are two separate cases - rather than trying to do two different runs from the same case, which requires careful saving and restoring of the appropriate files. One slight oddity: A TestStatus.log file appears in the clone directory (rather than the test's main case directory). At least for passing tests, this only includes a date/time stamp, so this didn't seem to be a problem. But it's possible that this could include more information if a test failed at a certain point. Presumably this is because I chdir to the case2 case directory before running case2 (I tried not doing this chdir, but this caused the run to fail) - and presumably there is some code somewhere that opens TestStatus.log in the current directory. I haven't tried to dig into this, because it didn't seem like a big deal to me, but can come back to it if others feel it's a problem. This also adds a new test based on this infrastructure (REP - a basic reproducibility test), and rewrites one existing test based on this infrastructure (LII). This also adds some unit tests written using the unittest module, and adds the capability for scripts_regression_tests to discover other unittests that may be distributed throughout the code base. Test suite: scripts_regression_tests.py on yellowstone Test baseline: Test namelist changes: Test status: bit for bit All passed except for the same 4 failures that I got in testing master: test_check_code (__main__.CheckCode) ... FAIL test_b_full (__main__.E_TestTestScheduler) ... FAIL test_save_timings (__main__.TestSaveTimings) ... FAIL test_full_system (__main__.Z_FullSystemTest) ... FAIL Fixes #214 Fixes #291 Fixes #292 Connects to #290 User interface changes?: none Code review: @jedwards4b @jgfouca @mvertens @rljacob
- Loading branch information
Showing
19 changed files
with
1,380 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,75 +1,35 @@ | ||
| """ | ||
| Implementation of the CIME LII test. This class inherits from SystemTestsCommon | ||
| Implementation of the CIME LII test. | ||
| This is a CLM specific test: | ||
| Verifies that namelist variable 'use_init_interp' works correctly | ||
| Verifies that interpolation of initial conditions onto an identical | ||
| configuration gives identical results: | ||
| (1) do a run with use_init_interp false (suffix base) | ||
| (2) do a run with use_init_interp true (suffix init_interp_on) | ||
| """ | ||
|
|
||
| import shutil, glob | ||
| from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo | ||
| from CIME.XML.standard_module_setup import * | ||
| from CIME.SystemTests.system_tests_common import SystemTestsCommon | ||
| from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| class LII(SystemTestsCommon): | ||
| class LII(SystemTestsCompareTwo): | ||
|
|
||
| def __init__(self, case): | ||
| """ | ||
| initialize a test object | ||
| """ | ||
| SystemTestsCommon.__init__(self, case) | ||
| SystemTestsCompareTwo.__init__(self, case, | ||
| separate_builds = False, | ||
| run_two_suffix = 'interp', | ||
| run_one_description = 'use_init_interp set to false', | ||
| run_two_description = 'use_init_interp set to true') | ||
|
|
||
| def _case_one_setup(self): | ||
| append_to_user_nl_files(caseroot = self._get_caseroot(), | ||
| component = "clm", | ||
| contents = "use_init_interp = .false.") | ||
|
|
||
| def _case_two_setup(self): | ||
| append_to_user_nl_files(caseroot = self._get_caseroot(), | ||
| component = "clm", | ||
| contents = "use_init_interp = .true.") | ||
|
|
||
| def build_phase(self, sharedlib_only=False, model_only=False): | ||
|
|
||
| # Make copies of the namelist files for each part of the test. Enclose the | ||
| # copies in conditionals so that we only do this namelist setup the first time | ||
| # the build script is invoked - otherwise, if the build is rerun, the namelist | ||
| # files would build up repeated instances of the setting of force_init_intep. | ||
| # | ||
| # Note the use of shell wildcards to make sure we apply these mods to | ||
| # multi-instance versions | ||
|
|
||
| if not os.path.exists("user_nl_nointerp"): | ||
| os.makedirs("user_nl_nointerp") | ||
| for filename in glob.glob(r'user_nl_clm*'): | ||
| shutil.copy(filename, os.path.join("user_nl_nointerp",filename)) | ||
| with open(os.path.join("user_nl_nointerp",filename), "a") as newfile: | ||
| newfile.write("use_init_interp = .false.") | ||
|
|
||
| if not os.path.exists("user_nl_interp"): | ||
| os.makedirs("user_nl_interp") | ||
| for filename in glob.glob(r'user_nl_clm*'): | ||
| shutil.copy(filename, os.path.join("user_nl_interp",filename)) | ||
| with open(os.path.join("user_nl_interp",filename), "a") as newfile: | ||
| newfile.write("use_init_interp = .true.") | ||
|
|
||
| self.clean_build() | ||
| self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) | ||
|
|
||
| def run_phase(self): | ||
| ''' | ||
| Do a run with init_interp false, a run with init_interp true and | ||
| compare | ||
| ''' | ||
| caseroot = self._case.get_value("CASEROOT") | ||
|
|
||
| self._case.set_value("CONTINUE_RUN",False) | ||
| self._case.set_value("REST_OPTION","none") | ||
| self._case.set_value("HIST_OPTION","$STOP_OPTION") | ||
| self._case.set_value("HIST_N","$STOP_N") | ||
| self._case.flush() | ||
| for user_nl_dir in ("nointerp", "interp"): | ||
| for filename in glob.glob(r'user_nl_%s/*'%user_nl_dir): | ||
| shutil.copy(filename, | ||
| os.path.join(caseroot,os.path.basename(filename))) | ||
|
|
||
| stop_n = self._case.get_value("STOP_N") | ||
| stop_option = self._case.get_value("STOP_OPTION") | ||
| logger.info("doing a %d %s initial test with init_interp set to %s, no restarts written" | ||
| % (stop_n, stop_option, user_nl_dir == "interp")) | ||
|
|
||
| self.run_indv(suffix=user_nl_dir) | ||
|
|
||
| self._component_compare_test("nointerp", "interp") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| """ | ||
| Implementation of the CIME REP test | ||
| This test verifies that two identical runs give bit-for-bit results | ||
| """ | ||
|
|
||
| from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo | ||
|
|
||
| class REP(SystemTestsCompareTwo): | ||
|
|
||
| def __init__(self, case): | ||
| SystemTestsCompareTwo.__init__(self, case, | ||
| separate_builds = False, | ||
| run_two_suffix = 'rep2') | ||
|
|
||
| def _case_one_setup(self): | ||
| pass | ||
|
|
||
| def _case_two_setup(self): | ||
| pass | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.