From 800027b8594dfb9372a0e3696303d02f11eb0404 Mon Sep 17 00:00:00 2001 From: Nicholas Hannah Date: Fri, 4 Nov 2016 22:43:06 +1100 Subject: [PATCH] Extend testing to handle dumping rho and sigma remapped diagnostics as well as z. #328 --- ice_ocean_SIS2/Baltic/MOM_input | 2 + src/MOM6 | 2 +- tools/tests/conftest.py | 23 ++++- tools/tests/dump_all_diagnostics.py | 11 ++- tools/tests/experiment.py | 78 +++++++++++---- tools/tests/test_diagnostic_output.py | 135 ++++++++++++++++++++++---- 6 files changed, 203 insertions(+), 48 deletions(-) diff --git a/ice_ocean_SIS2/Baltic/MOM_input b/ice_ocean_SIS2/Baltic/MOM_input index 03a561a49d..da0676f15f 100644 --- a/ice_ocean_SIS2/Baltic/MOM_input +++ b/ice_ocean_SIS2/Baltic/MOM_input @@ -61,6 +61,8 @@ NK = 63 ! [nondim] ! The number of model layers. DIAG_REMAP_Z_GRID_DEF = "FILE:OM3_zgrid.nc,zw,zt" +DIAG_REMAP_RHO_GRID_DEF = "UNIFORM" +DIAG_REMAP_SIGMA_GRID_DEF = "UNIFORM" ! === module MOM_verticalGrid === ! Parameters providing information about the vertical grid. diff --git a/src/MOM6 b/src/MOM6 index 32586a4ff0..50efe07593 160000 --- a/src/MOM6 +++ b/src/MOM6 @@ -1 +1 @@ -Subproject commit 32586a4ff048ec47603b4542d6d5664444af3fdc +Subproject commit 50efe07593840e1e59a153c83330a151d006b70f diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py index c71839bc4d..df4c69060f 100644 --- a/tools/tests/conftest.py +++ b/tools/tests/conftest.py @@ -23,8 +23,6 @@ def pytest_generate_tests(metafunc): Parameterize tests. Presently handles those that have 'exp' as an argument. """ - print("Calling pytest_generate_tests") - if 'exp' in metafunc.fixturenames: if metafunc.config.option.full: # Run tests on all experiments. @@ -49,13 +47,14 @@ def exp(request): """ exp = request.param - # Run the experiment to get latest code changes. This will do nothing if - # the experiment has already been run. + # Run the experiment to get latest code changes, and updates to the + # available_diags. This will do nothing if the experiment has already been + # run. exp.run() # Dump all available diagnostics, if they haven't been already. if not exp.has_dumped_diags: # Before dumping we delete old ones if they exist. - diags = exp.get_available_diags() + diags = exp.parse_available_diags() for d in diags: try: os.remove(d.output) @@ -66,6 +65,20 @@ def exp(request): exp.has_dumped_diags = True return exp + +@pytest.fixture(scope='session') +def exp_diags_not_dumped(): + + exp = experiment_dict['ice_ocean_SIS2/Baltic'] + + # Run the experiment to get latest code changes, and updates to the + # available_diags. This will do nothing if the experiment has already been + # run. + exp.run() + + return exp + + def restore_after_test(): """ Restore experiment state after running a test. diff --git a/tools/tests/dump_all_diagnostics.py b/tools/tests/dump_all_diagnostics.py index e7fa505888..c210dd5c69 100755 --- a/tools/tests/dump_all_diagnostics.py +++ b/tools/tests/dump_all_diagnostics.py @@ -29,14 +29,15 @@ def dump_diags(exp, diags): with open(os.path.join(exp.path, 'diag_table'), 'w') as f: print('All {} diags'.format(exp.name), file=f) print('1 1 1 0 0 0', file=f) - for d in diags: - print('"{}_{}", 0, "seconds", 1, "seconds",' \ - '"time"'.format(d.model, d.name), file=f) + for fname in list(set([d.filename for d in diags])): + print('"{}", 0, "seconds", 1, "seconds",' \ + '"time"'.format(fname), file=f) for d in diags: m = d.model n = d.name - print('"{}", "{}", "{}", "{}_{}", "all",' \ - '.false., "none", 2'.format(m, n, n, m, n), file=f) + fname = d.filename + print('"{}", "{}", "{}", "{}", "all",' \ + '.false., "none", 2'.format(m, n, n, fname, n), file=f) return exp.force_run() def main(): diff --git a/tools/tests/experiment.py b/tools/tests/experiment.py index a2c569d6e2..f97a5b14e2 100644 --- a/tools/tests/experiment.py +++ b/tools/tests/experiment.py @@ -5,6 +5,7 @@ import os import re import shlex +import random import subprocess as sp import run_config as rc from model import Model @@ -17,18 +18,33 @@ class Diagnostic: - def __init__(self, model, name, path): + def __init__(self, model, name, path, packed=True): self.model = model self.name = name self.full_name = '{}_{}'.format(model, name) - self.output = os.path.join(path, '00010101.{}.nc'.format(self.full_name)) + self.run_path = path + + # Hack to deal with FMS limitations, see https://github.com/NOAA-GFDL/FMS/issues/27 + # Use fewer different files for diagnostics. + if packed: + letter = self.name[0] + self.packed_filename = '{}_{}'.format(self.model, letter) + + self.unpacked_filename = '{}_{}'.format(self.model, self.name) + + if packed: + self.filename = self.packed_filename + else: + self.filename = self.unpacked_filename + + self.output = os.path.join(path, '00010101.{}.nc'.format(self.filename)) def __eq__(self, other): - return ((self.model, self.name, self.output) == - (other.model, other.name, other.output)) + return ((self.model, self.name) == + (other.model, other.name)) def __hash__(self): - return hash(self.model + self.name + self.output) + return hash(self.full_name) # Unfinished diagnostics are those which have been registered but have not been @@ -101,6 +117,7 @@ def __init__(self, id, platform='raijin', compiler='gnu', build='DEBUG', memory_ self.has_run = False # Another thing to avoid repeating. self.has_dumped_diags = False + self.diags_parsed = False def build_model(self): """ @@ -146,11 +163,19 @@ def force_run(self): return ret - def _parse_available_diags(self): + def parse_available_diags(self, packed=True): """ - Create a list of available diags for the experiment by parsing - available_diags.000001 and SIS.available_diags. + Return a list of the available diagnostics for this experiment by + parsing available_diags.000001 and SIS.available_diags. + + The 'packed' argument is used to pack many diagnostics into a few + output files. Without this each diagnostic is in it's own file. + + The experiment needs to have been run before calling this. """ + + assert self.has_run + mom_av_file = os.path.join(self.path, 'available_diags.000000') sis_av_file = os.path.join(self.path, 'SIS.available_diags') @@ -164,20 +189,14 @@ def _parse_available_diags(self): # Pull out the model name and variable name. matches = re.findall('^\"(\w+)\", \"(\w+)\".*$', f.read(), re.MULTILINE) - diags.extend([Diagnostic(m, d, self.path) for m, d in matches]) - return diags - - def get_available_diags(self): - """ - Return a list of the available diagnostics for this experiment. - - The experiment needs to have been run before calling this. - """ - - assert self.has_run + for m, d in matches: + if m[-5:] == '_zold': + diags.append(Diagnostic(m, d, self.path, packed=False)) + else: + diags.append(Diagnostic(m, d, self.path, packed)) # Lists of available and unfinished diagnostics. - self.available_diags = self._parse_available_diags() + self.available_diags = diags self.unfinished_diags = [Diagnostic(m, d, self.path) \ for m, d in _unfinished_diags] # Available diags is not what you think! Need to remove the unfinished @@ -187,8 +206,27 @@ def get_available_diags(self): # It helps with testing and human readability if this is sorted. self.available_diags.sort(key=lambda d: d.full_name) + self.diags_parsed = True + + return self.available_diags + + def get_diags(self): + + assert self.has_run + assert self.diags_parsed + return self.available_diags + def get_diags_dict(self): + + diags = self.get_diags() + + d = {} + for diag in diags: + d[diag.full_name] = diag + + return d + def get_unfinished_diags(self): """ Return a list of the unfinished diagnostics for this experiment. diff --git a/tools/tests/test_diagnostic_output.py b/tools/tests/test_diagnostic_output.py index 2fd763c381..998b518ace 100644 --- a/tools/tests/test_diagnostic_output.py +++ b/tools/tests/test_diagnostic_output.py @@ -8,12 +8,44 @@ import hashlib import pytest +from dump_all_diagnostics import dump_diags + DO_CHECKSUM_TEST = False -# FIXME: why are these so different between the old and new z remapped diags. -not_tested_z_diags = ['uh', 'vh'] +# FIXME: why are these different between the old and new z remapped diags. +# 1. The new z units for uh, vh are incorrect. +not_tested_z_diags = ['uh', 'vh', 'Kd_interface', 'Kd_itides', 'age'] + +def compare_rho_to_layer(coord_diags, layer_diags): + + layer_dict = {} + for d in layer_diags: + layer_dict[d.name] = d + + for da in coord_diags: + db = layer_dict[da.name] + + fa = Dataset(da.output) + fb = Dataset(db.output) + va = fa.variables[da.name][:] + vb = fb.variables[da.name][:] + + # Compare time axes. + assert np.array_equal(fa.variables['time'][:], fb.variables['time'][:]) + + # Compare the masks. Presently does not work for fields on interfaces. + if not 'i:point' in fa.variables[da.name].cell_methods: + assert np.array_equal(va.mask, vb.mask) + + if 'l:sum' in fa.variables[da.name].cell_methods: + # This will include, for example, uh, vh, h + assert np.allclose(np.sum(va, axis=1), np.sum(vb, axis=1), rtol=1e-2) + + fa.close() + fb.close() + -def compare_diags(diagA, diagB): +def compare_z_to_zold_diags(diagA, diagB): assert diagA.name == diagB.name @@ -61,11 +93,8 @@ def compare_diags(diagA, diagB): # Min and max should be close assert np.isclose(np.max(vs), np.max(vb), rtol=1e-2) assert np.isclose(np.min(vs), np.min(vb), rtol=1e-2) - - # Difference at every grid point - # assert np.allclose(vs, vb, rtol=1e-2) - # Overall difference + # Sums should be close if diagA.name in ['u', 'v', 'uo', 'vo']: # FIXME: investigate why these are so different. assert np.isclose(np.sum(vb), np.sum(vs), rtol=5e-2) @@ -75,6 +104,33 @@ def compare_diags(diagA, diagB): fa.close() fb.close() +def dump_diags_for_coord(exp, coord): + + assert coord == 'z' or coord == 'rho' or coord == 'sigma' + + def is_rho(diag): + return diag.model[-4:] == '_rho' + def is_sigma(diag): + return diag.model[-6:] == '_sigma' + def is_z(diag): + return diag.model[-2:] == '_z' + def is_layer(diag): + return diag.model == 'ocean_model' + + available_diags = exp.parse_available_diags(packed=False) + + if coord == 'rho': + coord_diags = filter(is_rho, available_diags) + if coord == 'z': + coord_diags = filter(is_z, available_diags) + if coord == 'sigma': + coord_diags = filter(is_sigma, available_diags) + layer_diags = filter(is_layer, available_diags) + + dump_diags(exp, coord_diags + layer_diags) + + return coord_diags, layer_diags + @pytest.mark.usefixtures('prepare_to_test') class TestDiagnosticOutput: @@ -85,15 +141,26 @@ def test_coverage(self, exp): """ # Check that none of the experiments unfinished diags have been # implemented, if so the unifinished_diags list should be updated. - assert(not any([os.path.exists(d.output) for d in exp.get_unfinished_diags()])) + for d in exp.get_unfinished_diags(): + if os.path.exists(d.output): + with Dataset(d.output) as f: + assert not f.variables.has_key(d.name) # Check that diags that should have been written out are. - assert(len(exp.get_available_diags()) > 0) - for d in exp.get_available_diags(): + assert(len(exp.get_diags()) > 0) + for d in exp.get_diags(): if not os.path.exists(d.output): print('Error: diagnostic output {} not found.'.format(d.output), file=sys.stderr) - assert(all([os.path.exists(d.output) for d in exp.get_available_diags()])) + assert os.path.exists(d.output) + + # Check that the output contains the expected variable + with Dataset(d.output) as f: + if not f.variables.has_key(d.name): + print('Error: diagnostic {} not found in {}.'.format(d.name, d.output), + file=sys.stderr) + assert f.variables.has_key(d.name) + def test_valid(self, exp): """ @@ -104,7 +171,7 @@ def test_valid(self, exp): - the variable contains data - that data doesn't contain NaNs. """ - for d in exp.get_available_diags(): + for d in exp.get_diags(): with Dataset(d.output) as f: assert(d.name in f.variables.keys()) data = f.variables[d.name][:].copy() @@ -114,8 +181,8 @@ def test_valid(self, exp): assert(not data.mask.all()) assert(not np.isnan(np.sum(data))) - @pytest.mark.z_remap - def test_z_remapping(self, exp): + @pytest.mark.cmp_z_remap + def test_cmp_z_zold_remapping(self, exp): """ Compare the new z space diagnostics (calculated using ALE remapping) to the old z remapped diagnostics. We expect them to be very similar. @@ -123,7 +190,7 @@ def test_z_remapping(self, exp): old_diags = [] new_diags = [] - for d in exp.get_available_diags(): + for d in exp.get_diags(): # Get new and old z diags. if d.model[-5:] == '_zold' and d.name[-6:] != '_xyave': old_diags.append(d) @@ -140,9 +207,43 @@ def test_z_remapping(self, exp): for old in old_diags: for new in new_diags: if new.name == old.name and new.name not in not_tested_z_diags: - compare_diags(old, new) + compare_z_to_zold_diags(old, new) break + @pytest.mark.rho_remap + def test_rho_remapping(self, exp_diags_not_dumped): + """ + Dump all rho diags into separate files. + """ + + rho_diags, layer_diags = dump_diags_for_coord(exp_diags_not_dumped, 'rho') + + compare_rho_to_layer(rho_diags, layer_diags) + + + @pytest.mark.sigma_remap + def test_sigma_remapping(self, exp_diags_not_dumped): + """ + Dump all sigma diags into separate files. + """ + + sigma_diags, layer_diags = dump_diags_for_coord(exp_diags_not_dumped, 'sigma') + + # FIXME. + # compare_sigma_to_layer(sigma_diags, layer_diags) + + + @pytest.mark.z_remap + def test_z_remapping(self, exp_diags_not_dumped): + """ + Dump all z diags into separate files. + """ + + z_diags, layer_diags = dump_diags_for_coord(exp_diags_not_dumped, 'z') + + # FIXME. + # compare_z_to_layer(z_diags, layer_diags) + @pytest.mark.skip(reason="This test is high maintenance. Also see DO_CHECKSUM_TEST.") def test_checksums(self, exp): @@ -154,7 +255,7 @@ def test_checksums(self, exp): checksum_file = os.path.join(exp.path, 'diag_checksums.txt') tmp_file = os.path.join(exp.path, 'tmp_diag_checksums.txt') new_checksums = '' - for d in exp.get_available_diags(): + for d in exp.get_diags(): # hash the diagnostic data in the output file with Dataset(d.output) as f: checksum = hashlib.sha1(f.variables[d.name][:].data.tobytes()).hexdigest()