From 9c4430a9080c0aa001ab2eb5ead5046b26e0279d Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 17 Jun 2024 18:39:14 -0600 Subject: [PATCH 1/8] initial commit --- testing/functional_tests.cfg | 15 ++ testing/run_fates_tests.py | 467 +++++++++++++++++------------------ 2 files changed, 237 insertions(+), 245 deletions(-) create mode 100644 testing/functional_tests.cfg diff --git a/testing/functional_tests.cfg b/testing/functional_tests.cfg new file mode 100644 index 0000000000..499c94f39e --- /dev/null +++ b/testing/functional_tests.cfg @@ -0,0 +1,15 @@ +[allometry] +test_dir = fates_allom_ftest +test_exe = FATES_allom_exe +out_file = allometry_out.nc +use_param_file = True +other_args = [] +plotting_function = plot_allometry_dat + +[quadratic] +test_dir = fates_math_ftest +test_exe = FATES_math_exe +out_file = quad_out.nc +use_param_file = False +other_args = [] +plotting_function = plot_quadratic_dat \ No newline at end of file diff --git a/testing/run_fates_tests.py b/testing/run_fates_tests.py index ea2cf5b0b1..7bc9608b0d 100755 --- a/testing/run_fates_tests.py +++ b/testing/run_fates_tests.py @@ -27,6 +27,7 @@ """ import os +import configparser import argparse import matplotlib.pyplot as plt from build_fortran_tests import build_unit_tests, build_exists @@ -40,233 +41,12 @@ from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order # Constants for this script +_DEFAULT_CONFIG_FILE = "functional_tests.cfg" _DEFAULT_CDL_PATH = os.path.abspath("../parameter_files/fates_params_default.cdl") _CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") _TEST_NAME = "fates_tests" _TEST_SUB_DIR = "testing" -# Dictionary with needed constants for running the executables and reading in the -# output files - developers who add tests should add things here. - -# NOTE: if the functional test you write requires a parameter file read in as a -# command-line argument, this should be the *first* (or only) argument in the -# command-line argument list -_ALL_TESTS_DICT = { - "allometry": { - "test_dir": "fates_allom_ftest", - "test_exe": "FATES_allom_exe", - "out_file": "allometry_out.nc", - "has_unit_test": False, - "use_param_file": True, - "other_args": [], - "plotting_function": plot_allometry_dat, - }, - "quadratic": { - "test_dir": "fates_math_ftest", - "test_exe": "FATES_math_exe", - "out_file": "quad_out.nc", - "has_unit_test": False, - "use_param_file": False, - "other_args": [], - "plotting_function": plot_quadratic_dat, - }, - "fire_weather":{ - "test_dir": "fates_fire_weather_utest", - "test_exe": None, - "out_file": None, - "has_unit_test": True, - "use_param_file": False, - "other_args": [], - "plotting_function": None, - } - } - -def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): - """Run the generated Fortran executables - - Args: - build_dir (str): full path to build directory - run_dir (str): full path to run directory - test_dir (str): test directory within the run directory - test_exe (str): test executable to run - args ([str]): arguments for executable - """ - - # move executable to run directory - exe_path = os.path.join(build_dir, _TEST_SUB_DIR, test_dir, test_exe) - copy_file(exe_path, run_dir) - - # run the executable - new_exe_path = os.path.join(run_dir, test_exe) - run_command = [new_exe_path] - run_command.extend(args) - - os.chdir(run_dir) - out = run_cmd_no_fail(" ".join(run_command), combine_output=True) - print(out) - -def make_plotdirs(run_dir, test_dict): - """Create plotting directories if they don't already exist - - Args: - run_dir (str): full path to run directory - test_dict (dict): dictionary of test to run - """ - # make main plot directory - plot_dir = os.path.join(run_dir, 'plots') - if not os.path.isdir(plot_dir): - os.mkdir(plot_dir) - - # make sub-plot directories - for test in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - test_dict.items())): - sub_dir = os.path.join(plot_dir, test) - if not os.path.isdir(sub_dir): - os.mkdir(sub_dir) - -def create_param_file(param_file, run_dir): - """Creates and/or move the default or input parameter file to the run directory - Creates a netcdf file from a cdl file if a cdl file is supplied - - Args: - param_file (str): path to parmaeter file - run_dir (str): full path to run directory - - Raises: - RuntimeError: Supplied parameter file is not netcdf (.cd) or cdl (.cdl) - - Returns: - str: full path to new parameter file name/location - """ - if param_file is None: - print("Using default parameter file.") - param_file = _DEFAULT_CDL_PATH - param_file_update = create_nc_file(param_file, run_dir) - else: - print(f"Using parameter file {param_file}.") - file_suffix = os.path.basename(param_file).split(".")[-1] - if file_suffix == 'cdl': - param_file_update = create_nc_file(param_file, run_dir) - elif file_suffix == "nc": - param_file_update = copy_file(param_file, run_dir) - else: - raise RuntimeError("Must supply parameter file with .cdl or .nc ending.") - - return param_file_update - -def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_dir, - make_j, param_file, save_figs, test_dict): - """Builds and runs the fates tests - - Args: - clean (bool): whether or not to clean the build directory - verbose_make (bool): whether or not to run make with verbose output - build_tests (bool): whether or not to build the exectuables - run_executables (bool): whether or not to run the executables - build_dir (str): build directory - run_dir (str): run directory - make_j (int): number of processors for the build - param_file (str): input FATES parameter file - save_figs (bool): whether or not to write figures to file - test_dict (dict): dictionary of tests to run - """ - - # absolute path to desired build directory - build_dir_path = os.path.abspath(build_dir) - - # absolute path to desired run directory - run_dir_path = os.path.abspath(run_dir) - - # make run directory if it doesn't already exist - if not os.path.isdir(run_dir_path): - os.mkdir(run_dir_path) - - # create plot directories if we need to - if save_figs: - make_plotdirs(os.path.abspath(run_dir), test_dict) - - # move parameter file to correct location (creates nc file if cdl supplied) - param_file = create_param_file(param_file, run_dir) - - # compile code - if build_tests: - build_unit_tests(build_dir, _TEST_NAME, _CMAKE_BASE_DIR, make_j, clean=clean, - verbose=verbose_make) - - # run executables for each test in test list - if run_executables: - print("Running executables") - # we don't run executables for only pfunit tests - for attributes in dict(filter(lambda pair: pair[1]['test_exe'] is not None, - test_dict.items())).values(): - # prepend parameter file (if required) to argument list - args = attributes['other_args'] - if attributes['use_param_file']: - args.insert(0, param_file) - # run - run_fortran_exectuables(build_dir_path, attributes['test_dir'], - attributes['test_exe'], run_dir_path, args) - - # run unit tests - for test, attributes in dict(filter(lambda pair: pair[1]['has_unit_test'], - test_dict.items())).items(): - print(f"Running unit tests for {test}.") - - test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) - ctest_command = ["ctest", "--output-on-failure"] - output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, - combine_output=True) - print(output) - - # plot output for relevant tests - for test, attributes in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - test_dict.items())).items(): - attributes['plotting_function'](run_dir_path, - attributes['out_file'], save_figs, - os.path.join(run_dir_path, 'plots', test)) - # show plots - plt.show() - -def out_file_exists(run_dir, out_file): - """Checks to see if the file out_file exists in the run_dir - - Args: - run_dir (str): full path to run directory - out_file (str): output file name - - Returns: - bool: yes/no file exists in correct location - """ - return os.path.isfile(os.path.join(run_dir, out_file)) - -def parse_test_list(test_string): - """Parses the input test list and checks for errors - - Args: - test (str): user-supplied comma-separated list of test names - - Returns: - dictionary: filtered dictionary of tests to run - - Raises: - RuntimeError: Invalid test name supplied - """ - valid_test_names = _ALL_TESTS_DICT.keys() - - if test_string != "all": - test_list = test_string.split(',') - for test in test_list: - if test not in valid_test_names: - raise argparse.ArgumentTypeError("Invalid test supplied, \n" - "must supply one of:\n" - f"{', '.join(valid_test_names)}\n" - "or do not supply a test name to run all tests.") - test_dict = {key: _ALL_TESTS_DICT[key] for key in test_list} - else: - test_dict = _ALL_TESTS_DICT - - return test_dict - def commandline_args(): """Parse and return command-line arguments""" @@ -275,7 +55,7 @@ def commandline_args(): Typical usage: - ./run_fates_tests -f parameter_file.nc + ./run_fates_tests -t allometry """ parser = argparse.ArgumentParser( @@ -360,8 +140,8 @@ def commandline_args(): "-t", "--test-list", action="store", - dest="test_dict", - type=parse_test_list, + dest="test_list", + type=str, default="all", help="Test(s) to run. Comma-separated list of test names, or 'all'\n" "for all tests. If not supplied, will run all tests." @@ -373,6 +153,26 @@ def commandline_args(): return args +def check_arg_validity(args): + """Checks validity of input script arguments + + Args: + args (parse_args): input arguments + """ + # check to make sure parameter file exists and is one of the correct forms + check_param_file(args.parameter_file) + + # make sure build directory exists + if args.skip_build: + if args.verbose_make: + raise argparse.ArgumentError(None, "Can't run verbose make and skip build.\n" + "Re-run script without --skip-build") + check_build_dir(args.build_dir, args.test_dict) + + # make sure relevant output files exist: + if args.skip_run_executables: + check_out_files(args.run_dir, args.test_dict) + def check_param_file(param_file): """Checks to see if param_file exists and is of the correct form (.nc or .cdl) @@ -388,7 +188,7 @@ def check_param_file(param_file): raise argparse.ArgumentError(None, "Must supply parameter file with .cdl or .nc ending.") if not os.path.isfile(param_file): raise argparse.ArgumentError(None, f"Cannot find file {param_file}.") - + def check_build_dir(build_dir, test_dict): """Checks to see if all required build directories and executables are present @@ -403,7 +203,7 @@ def check_build_dir(build_dir, test_dict): if not build_exists(build_dir, attributes['test_dir'], attributes['test_exe']): raise argparse.ArgumentError(None, "Build directory or executable does not exist.\n" "Re-run script without --skip-build.") - + def check_out_files(run_dir, test_dict): """Checks to see that required output files are present in the run directory @@ -416,42 +216,219 @@ def check_out_files(run_dir, test_dict): """ for test, attributes in dict(filter(lambda pair: pair[1]['out_file'] is not None, test_dict.items())).items(): - if not out_file_exists(os.path.abspath(run_dir), attributes['out_file']): + if not os.path.isfile(os.path.join(os.path.abspath(run_dir), attributes['out_file'])): raise argparse.ArgumentError(None, f"Required file for {test} test does not exist.\n" "Re-run script without --skip-run.") -def check_arg_validity(args): - """Checks validity of input script arguments +def parse_test_list(full_test_dict, test_string): + """Parses the input test list and checks for errors Args: - args (parse_args): input arguments + test (str): user-supplied comma-separated list of test names + + Returns: + dictionary: filtered dictionary of tests to run + + Raises: + RuntimeError: Invalid test name supplied """ - # check to make sure parameter file exists and is one of the correct forms - check_param_file(args.parameter_file) + valid_test_names = full_test_dict.keys() - # make sure build directory exists - if args.skip_build: - if args.verbose_make: - raise argparse.ArgumentError(None, "Can't run verbose make and skip build.\n" - "Re-run script without --skip-build") - check_build_dir(args.build_dir, args.test_dict) + if test_string != "all": + test_list = test_string.split(',') + for test in test_list: + if test not in valid_test_names: + raise argparse.ArgumentTypeError("Invalid test supplied, \n" + "must supply one of:\n" + f"{', '.join(valid_test_names)}\n" + "or do not supply a test name to run all tests.") + test_dict = {key: full_test_dict[key] for key in test_list} + else: + test_dict = full_test_dict - # make sure relevant output files exist: - if args.skip_run_executables: - check_out_files(args.run_dir, args.test_dict) + return test_dict + +def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): + """Run the generated Fortran executables + + Args: + build_dir (str): full path to build directory + run_dir (str): full path to run directory + test_dir (str): test directory within the run directory + test_exe (str): test executable to run + args ([str]): arguments for executable + """ + # move executable to run directory + exe_path = os.path.join(build_dir, _TEST_SUB_DIR, test_dir, test_exe) + copy_file(exe_path, run_dir) + + # run the executable + new_exe_path = os.path.join(run_dir, test_exe) + run_command = [new_exe_path] + run_command.extend(args) + + os.chdir(run_dir) + out = run_cmd_no_fail(" ".join(run_command), combine_output=True) + print(out) + +def make_plotdirs(run_dir, test_dict): + """Create plotting directories if they don't already exist + + Args: + run_dir (str): full path to run directory + test_dict (dict): dictionary of test to run + """ + # make main plot directory + plot_dir = os.path.join(run_dir, 'plots') + if not os.path.isdir(plot_dir): + os.mkdir(plot_dir) + + # make sub-plot directories + for test in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, + test_dict.items())): + sub_dir = os.path.join(plot_dir, test) + if not os.path.isdir(sub_dir): + os.mkdir(sub_dir) + +def create_param_file(param_file, run_dir): + """Creates and/or move the default or input parameter file to the run directory + Creates a netcdf file from a cdl file if a cdl file is supplied + + Args: + param_file (str): path to parmaeter file + run_dir (str): full path to run directory + + Raises: + RuntimeError: Supplied parameter file is not netcdf (.cd) or cdl (.cdl) + + Returns: + str: full path to new parameter file name/location + """ + if param_file is None: + print("Using default parameter file.") + param_file = _DEFAULT_CDL_PATH + param_file_update = create_nc_file(param_file, run_dir) + else: + print(f"Using parameter file {param_file}.") + file_suffix = os.path.basename(param_file).split(".")[-1] + if file_suffix == 'cdl': + param_file_update = create_nc_file(param_file, run_dir) + elif file_suffix == "nc": + param_file_update = copy_file(param_file, run_dir) + else: + raise RuntimeError("Must supply parameter file with .cdl or .nc ending.") + + return param_file_update + +def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_dir, + make_j, param_file, save_figs, test_dict): + """Builds and runs the fates tests + + Args: + clean (bool): whether or not to clean the build directory + verbose_make (bool): whether or not to run make with verbose output + build_tests (bool): whether or not to build the exectuables + run_executables (bool): whether or not to run the executables + build_dir (str): build directory + run_dir (str): run directory + make_j (int): number of processors for the build + param_file (str): input FATES parameter file + save_figs (bool): whether or not to write figures to file + test_dict (dict): dictionary of tests to run + """ + + # absolute path to desired build directory + build_dir_path = os.path.abspath(build_dir) + + # absolute path to desired run directory + run_dir_path = os.path.abspath(run_dir) + + # make run directory if it doesn't already exist + if not os.path.isdir(run_dir_path): + os.mkdir(run_dir_path) + + # create plot directories if we need to + if save_figs: + make_plotdirs(os.path.abspath(run_dir), test_dict) + + # move parameter file to correct location (creates nc file if cdl supplied) + param_file = create_param_file(param_file, run_dir) + + # compile code + if build_tests: + build_unit_tests(build_dir, _TEST_NAME, _CMAKE_BASE_DIR, make_j, clean=clean, + verbose=verbose_make) + + # run executables for each test in test list + if run_executables: + print("Running executables") + # we don't run executables for only pfunit tests + for attributes in dict(filter(lambda pair: pair[1]['test_exe'] is not None, + test_dict.items())).values(): + # prepend parameter file (if required) to argument list + args = attributes['other_args'] + if attributes['use_param_file']: + args.insert(0, param_file) + # run + run_fortran_exectuables(build_dir_path, attributes['test_dir'], + attributes['test_exe'], run_dir_path, args) + + # run unit tests + for test, attributes in dict(filter(lambda pair: pair[1]['has_unit_test'], + test_dict.items())).items(): + print(f"Running unit tests for {test}.") + + test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) + ctest_command = ["ctest", "--output-on-failure"] + output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, + combine_output=True) + print(output) + + # plot output for relevant tests + for test, attributes in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, + test_dict.items())).items(): + attributes['plotting_function'](run_dir_path, + attributes['out_file'], save_figs, + os.path.join(run_dir_path, 'plots', test)) + # show plots + plt.show() + +def config_to_dict(config_file): + """Convert a config file to a python dictionary + + Args: + config_file (str): full path to config file + + Returns: + dictionary: dictionary of config file + """ + config = configparser.ConfigParser() + config.read(config_file) + + dictionary = {} + for section in config.sections(): + dictionary[section] = {} + for option in config.options(section): + dictionary[section][option] = config.get(section, option) + + return dictionary + def main(): """Main script Reads in command-line arguments and then runs the tests. """ - + + functional_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) + args = commandline_args() + test_dict = parse_test_list(functional_test_dict, args.test_list) build = not args.skip_build run = not args.skip_run_executables run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, - args.make_j, args.parameter_file, args.save_figs, args.test_dict) + args.make_j, args.parameter_file, args.save_figs, test_dict) if __name__ == "__main__": - main() + main() \ No newline at end of file From e74262f483f564a02a4384ede8abbb7ed2e02628 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Wed, 19 Jun 2024 12:14:50 -0600 Subject: [PATCH 2/8] some updates --- testing/build_fortran_tests.py | 223 +++++++++--------- .../allometry/allometry_test.py | 5 + testing/functional_tests.cfg | 4 +- testing/path_utils.py | 61 +++-- testing/run_fates_tests.py | 189 +++++++-------- testing/testing_classes.py | 22 ++ testing/utils.py | 102 ++++++-- 7 files changed, 344 insertions(+), 262 deletions(-) create mode 100644 testing/functional_testing/allometry/allometry_test.py create mode 100644 testing/testing_classes.py diff --git a/testing/build_fortran_tests.py b/testing/build_fortran_tests.py index b495f25020..35f5c51350 100644 --- a/testing/build_fortran_tests.py +++ b/testing/build_fortran_tests.py @@ -3,7 +3,7 @@ """ import os import shutil -from utils import add_cime_lib_to_path +from path_utils import add_cime_lib_to_path add_cime_lib_to_path() @@ -13,81 +13,50 @@ from CIME.BuildTools.configure import configure, FakeCase # pylint: disable=wrong-import-position,import-error,wrong-import-order from CIME.XML.env_mach_specific import EnvMachSpecific # pylint: disable=wrong-import-position,import-error,wrong-import-order +# constants for this script _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../cime") +_MPI_LIBRARY = "mpi-serial" -def run_cmake(test_name, test_dir, pfunit_path, netcdf_c_path, netcdf_f_path, cmake_args): - """Run cmake for the fortran unit tests - Arguments: - test_name (str) - name for output messages - test_dir (str) - directory to run Cmake in - pfunit_path (str) - path to pfunit - netcdf_c_path (str) - path to netcdf - netcdf_f_path (str) - path to netcdff - clean (bool) - clean the build - """ - if not os.path.isfile("CMakeCache.txt"): - print(f"Running cmake for {test_name}.") - - # directory with cmake modules - cmake_module_dir = os.path.abspath(os.path.join(_CIMEROOT, "CIME", "non_py", - "src", "CMake")) - - # directory with genf90 - genf90_dir = os.path.join(_CIMEROOT, "CIME", "non_py", "externals", "genf90") - - cmake_command = [ - "cmake", - "-C Macros.cmake", - test_dir, - f"-DCIMEROOT={_CIMEROOT}", - f"-DSRC_ROOT={get_src_root()}", - f"-DCIME_CMAKE_MODULE_DIRECTORY={cmake_module_dir}", - "-DCMAKE_BUILD_TYPE=CESM_DEBUG", - f"-DCMAKE_PREFIX_PATH={pfunit_path}", - "-DUSE_MPI_SERIAL=ON", - "-DENABLE_GENF90=ON", - f"-DCMAKE_PROGRAM_PATH={genf90_dir}" - ] - - if netcdf_c_path is not None: - cmake_command.append(f"-DNETCDF_C_PATH={netcdf_c_path}") - - if netcdf_f_path is not None: - cmake_command.append(f"-DNETCDF_F_PATH={netcdf_f_path}") - - cmake_command.extend(cmake_args.split(" ")) - - run_cmd_no_fail(" ".join(cmake_command), combine_output=True) - -def find_library(caseroot, cmake_args, lib_string): - """Find the library installation we'll be using, and print its path +def build_tests(build_dir:str, cmake_directory:str, make_j:int, clean:bool=False, + verbose:bool=False): + """Builds the test executables Args: - caseroot (str): Directory with pfunit macros - cmake_args (str): The cmake args used to invoke cmake - (so that we get the correct makefile vars) + build_dir (str): build directory + cmake_directory (str): directory where the make CMakeLists.txt file is + make_j (int): number of processes to use for make + clean (bool, optional): whether or not to clean the build first. Defaults to False. + verbose (bool, optional): whether or not to run make with verbose output. Defaults to False. """ - with CmakeTmpBuildDir(macroloc=caseroot) as cmaketmp: - all_vars = cmaketmp.get_makefile_vars(cmake_args=cmake_args) + # create the build directory + full_build_path = prep_build_dir(build_dir, clean=clean) - all_vars_list = all_vars.splitlines() - for all_var in all_vars_list: - if ":=" in all_var: - expect(all_var.count(":=") == 1, f"Bad makefile: {all_var}") - varname, value = [item.strip() for item in all_var.split(":=")] - if varname == lib_string: - return value + # get cmake args and the pfunit and netcdf paths + cmake_args = get_extra_cmake_args(full_build_path, _MPI_LIBRARY) + pfunit_path = find_library(full_build_path, cmake_args, "PFUNIT_PATH") - expect(False, f"{lib_string} not found for this machine and compiler") + if not "NETCDF" in os.environ: + netcdf_c_path = find_library(full_build_path, cmake_args, "NETCDF_C_PATH") + netcdf_f_path = find_library(full_build_path, cmake_args, "NETCDF_FORTRAN_PATH") + else: + netcdf_c_path = None + netcdf_f_path = None - return None + # change into the build dir + os.chdir(full_build_path) -def prep_build_dir(build_dir, clean): - """Create (if necessary) build directory and clean contents (if asked to) + # run cmake and make + run_cmake(cmake_directory, pfunit_path, netcdf_c_path, netcdf_f_path, cmake_args) + run_make(make_j, clean=clean, verbose=verbose) + +def prep_build_dir(build_dir:str, clean:bool) -> str: + """Creates (if necessary) build directory and cleans contents (if asked to) Args: build_dir (str): build directory name clean (bool): whether or not to clean contents + Returns: + str: full build path """ # create the build directory @@ -115,7 +84,7 @@ def clean_cmake_files(): cwd_contents = os.listdir(os.getcwd()) - # Clear contents to do with cmake cache + # clear contents to do with cmake cache for file in cwd_contents: if ( file in ("Macros.cmake", "env_mach_specific.xml") @@ -123,15 +92,17 @@ def clean_cmake_files(): or file.startswith(".env_mach_specific") ): os.remove(file) - -def get_extra_cmake_args(build_dir, mpilib): + +def get_extra_cmake_args(build_dir:str, mpilib:str) -> str: """Makes a fake case to grab the required cmake arguments Args: build_dir (str): build directory name mpilib (str): MPI library name + Returns: + str: space-separated list of cmake arguments """ # get the machine objects file - machobj = Machines() # this is different? + machobj = Machines() # get compiler compiler = machobj.get_default_compiler() @@ -139,9 +110,7 @@ def get_extra_cmake_args(build_dir, mpilib): # get operating system os_ = machobj.get_value("OS") - # Create the environment, and the Macros.cmake file - # - # + # create the environment, and the Macros.cmake file configure( machobj, build_dir, @@ -158,6 +127,8 @@ def get_extra_cmake_args(build_dir, mpilib): # make a fake case fake_case = FakeCase(compiler, mpilib, True, "nuopc", threading=False) machspecific.load_env(fake_case) + + # create cmake argument list with information from the fake case and machine object cmake_args_list = [ f"-DOS={os_}", f"-DMACH={machobj.get_machine_name()}", @@ -172,18 +143,82 @@ def get_extra_cmake_args(build_dir, mpilib): return cmake_args -def run_make(name, make_j, clean=False, verbose=False): +def find_library(caseroot:str, cmake_args:str, lib_string:str) -> str: + """Find the library installation we'll be using, and return its path + + Args: + caseroot (str): Directory with pfunit macros + cmake_args (str): The cmake args used to invoke cmake + (so that we get the correct makefile vars) + Returns: + str: full path to library installation + """ + with CmakeTmpBuildDir(macroloc=caseroot) as cmaketmp: + all_vars = cmaketmp.get_makefile_vars(cmake_args=cmake_args) + + all_vars_list = all_vars.splitlines() + for all_var in all_vars_list: + if ":=" in all_var: + expect(all_var.count(":=") == 1, f"Bad makefile: {all_var}") + varname, value = [item.strip() for item in all_var.split(":=")] + if varname == lib_string: + return value + + expect(False, f"{lib_string} not found for this machine and compiler") + + return None + +def run_cmake(test_dir:str, pfunit_path:str, netcdf_c_path:str, netcdf_f_path:str, cmake_args:str): + """Run cmake for the fortran unit tests + Arguments: + test_dir (str) - directory to run Cmake in + pfunit_path (str) - path to pfunit + netcdf_c_path (str) - path to netcdf + netcdf_f_path (str) - path to netcdff + cmake_args (str) - extra arguments to Cmake + """ + if not os.path.isfile("CMakeCache.txt"): + + # directory with cmake modules + cmake_module_dir = os.path.abspath(os.path.join(_CIMEROOT, "CIME", "non_py", + "src", "CMake")) + # directory with genf90 + genf90_dir = os.path.join(_CIMEROOT, "CIME", "non_py", "externals", "genf90") + + cmake_command = [ + "cmake", + "-C Macros.cmake", + test_dir, + f"-DCIMEROOT={_CIMEROOT}", + f"-DSRC_ROOT={get_src_root()}", + f"-DCIME_CMAKE_MODULE_DIRECTORY={cmake_module_dir}", + "-DCMAKE_BUILD_TYPE=CESM_DEBUG", + f"-DCMAKE_PREFIX_PATH={pfunit_path}", + "-DUSE_MPI_SERIAL=ON", + "-DENABLE_GENF90=ON", + f"-DCMAKE_PROGRAM_PATH={genf90_dir}" + ] + + if netcdf_c_path is not None: + cmake_command.append(f"-DNETCDF_C_PATH={netcdf_c_path}") + + if netcdf_f_path is not None: + cmake_command.append(f"-DNETCDF_F_PATH={netcdf_f_path}") + + cmake_command.extend(cmake_args.split(" ")) + + print("Running cmake for all tests.") + + run_cmd_no_fail(" ".join(cmake_command), combine_output=True) + +def run_make(make_j:int, clean:bool=False, verbose:bool=False): """Run make in current working directory Args: - name (str): Name for output messages make_j (int): number of processes to use for make clean (bool, optional): whether or not to clean Defaults to False. verbose (bool, optional): verbose error logging for make Defaults to False. """ - - print(f"Running make for {name}.") - if clean: run_cmd_no_fail("make clean") @@ -191,16 +226,20 @@ def run_make(name, make_j, clean=False, verbose=False): if verbose: make_command.append("VERBOSE=1") + + print("Running make for all tests.") run_cmd_no_fail(" ".join(make_command), combine_output=True) -def build_exists(build_dir, test_dir, test_exe=None): +def build_exists(build_dir:str, test_dir:str, test_exe:str=None) -> bool: """Checks to see if the build directory and associated executables exist. Args: build_dir (str): build directory test_dir (str): test directory test_exe (str): test executable + Returns: + bool: whether or not build directory and associated executables exist """ build_path = os.path.abspath(build_dir) @@ -215,35 +254,3 @@ def build_exists(build_dir, test_dir, test_exe=None): return False return True - -def build_unit_tests(build_dir, name, cmake_directory, make_j, clean=False, verbose=False): - """Build the unit test executables - - Args: - build_dir (str): build directory - name (str): name for set of tests - cmake_directory (str): directory where the make CMakeLists.txt file is - make_j (int): number of processes to use for make - clean (bool, optional): whether or not to clean the build first. Defaults to False. - verbose (bool, optional): whether or not to run make with verbose output. Defaults to False. - """ - # create the build directory - full_build_path = prep_build_dir(build_dir, clean=clean) - - # get cmake args and the pfunit and netcdf paths - cmake_args = get_extra_cmake_args(full_build_path, "mpi-serial") - pfunit_path = find_library(full_build_path, cmake_args, "PFUNIT_PATH") - - if not "NETCDF" in os.environ: - netcdf_c_path = find_library(full_build_path, cmake_args, "NETCDF_C_PATH") - netcdf_f_path = find_library(full_build_path, cmake_args, "NETCDF_FORTRAN_PATH") - else: - netcdf_c_path = None - netcdf_f_path = None - - # change into the build dir - os.chdir(full_build_path) - - # run cmake and make - run_cmake(name, cmake_directory, pfunit_path, netcdf_c_path, netcdf_f_path, cmake_args) - run_make(name, make_j, clean=clean, verbose=verbose) diff --git a/testing/functional_testing/allometry/allometry_test.py b/testing/functional_testing/allometry/allometry_test.py new file mode 100644 index 0000000000..fe800b91dd --- /dev/null +++ b/testing/functional_testing/allometry/allometry_test.py @@ -0,0 +1,5 @@ +from testing_classes import FunctionalTest + +class AllometryTest(FunctionalTest): + def __init__() + \ No newline at end of file diff --git a/testing/functional_tests.cfg b/testing/functional_tests.cfg index 499c94f39e..3c16e8cb1c 100644 --- a/testing/functional_tests.cfg +++ b/testing/functional_tests.cfg @@ -4,12 +4,10 @@ test_exe = FATES_allom_exe out_file = allometry_out.nc use_param_file = True other_args = [] -plotting_function = plot_allometry_dat [quadratic] test_dir = fates_math_ftest test_exe = FATES_math_exe out_file = quad_out.nc use_param_file = False -other_args = [] -plotting_function = plot_quadratic_dat \ No newline at end of file +other_args = [] \ No newline at end of file diff --git a/testing/path_utils.py b/testing/path_utils.py index 85aea6085a..3a5e533606 100644 --- a/testing/path_utils.py +++ b/testing/path_utils.py @@ -4,48 +4,57 @@ import os import sys -# Path to the root directory of FATES, based on the path of this file -# Note: It's important that this NOT end with a trailing slash; +# path to the root directory of FATES, based on the path of this file +# it's important that this NOT end with a trailing slash _FATES_ROOT = os.path.normpath( os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) ) -def path_to_fates_root(): - """Returns the path to the root directory of FATES""" - return _FATES_ROOT +def add_cime_lib_to_path() -> str: + """Adds the CIME python library to the python path, to allow importing + modules from that library + + Returns: + str: path to top-level cime directory + """ + cime_path = path_to_cime() + prepend_to_python_path(cime_path) + + cime_lib_path = os.path.join(cime_path, "CIME", "Tools") + prepend_to_python_path(cime_lib_path) + + return cime_path -def path_to_cime(): +def path_to_cime() -> str: """Returns the path to cime, if it can be found - Raises a RuntimeError if it cannot be found + Raises: + RuntimeError: can't find path to cime + Returns: + str: full path to cime """ cime_path = os.path.join(path_to_fates_root(), "../../cime") if os.path.isdir(cime_path): return cime_path raise RuntimeError("Cannot find cime.") -def prepend_to_python_path(path): - """Adds the given path to python's sys.path if it isn't already in the path +def path_to_fates_root(): + """Returns Returns the path to the root directory of FATES - The path is added near the beginning, so that it takes precedence over existing - entries in the path + Returns: + str: path to the root directory of FATES """ - if not path in sys.path: - # Insert at location 1 rather than 0, because 0 is special - sys.path.insert(1, path) - -def add_cime_lib_to_path(): - """Adds the CIME python library to the python path, to allow importing - modules from that library + return _FATES_ROOT - Returns the path to the top-level cime directory +def prepend_to_python_path(path:str): + """Adds the given path to python's sys.path if not already there + Path is added near the beginning, so that it takes precedence over existing + entries in path. - For documentation on standalone_only: See documentation in - path_to_cime + Args: + path (str): input path """ - cime_path = path_to_cime() - prepend_to_python_path(cime_path) - cime_lib_path = os.path.join(cime_path, "CIME", "Tools") - prepend_to_python_path(cime_lib_path) - return cime_path + if not path in sys.path: + # insert at location 1 rather than 0, because 0 is special + sys.path.insert(1, path) diff --git a/testing/run_fates_tests.py b/testing/run_fates_tests.py index 7bc9608b0d..9d7cc28801 100755 --- a/testing/run_fates_tests.py +++ b/testing/run_fates_tests.py @@ -27,12 +27,11 @@ """ import os -import configparser import argparse import matplotlib.pyplot as plt -from build_fortran_tests import build_unit_tests, build_exists +from build_fortran_tests import build_tests, build_exists from path_utils import add_cime_lib_to_path -from utils import copy_file, create_nc_file +from utils import copy_file, create_nc_from_cdl, config_to_dict, str_to_list from functional_testing.allometry.allometry_plotting import plot_allometry_dat from functional_testing.math_utils.math_plotting import plot_quadratic_dat @@ -40,11 +39,10 @@ from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order -# Constants for this script +# constants for this script _DEFAULT_CONFIG_FILE = "functional_tests.cfg" _DEFAULT_CDL_PATH = os.path.abspath("../parameter_files/fates_params_default.cdl") _CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") -_TEST_NAME = "fates_tests" _TEST_SUB_DIR = "testing" def commandline_args(): @@ -77,7 +75,7 @@ def commandline_args(): "-b", "--build-dir", type=str, - default="../_build", + default=os.path.join(_CMAKE_BASE_DIR, "_build"), help="Directory where tests are built.\n" "Will be created if it does not exist.\n", ) @@ -86,7 +84,7 @@ def commandline_args(): "-r", "--run-dir", type=str, - default="../_run", + default=os.path.join(_CMAKE_BASE_DIR, "_run"), help="Directory where tests are run.\n" "Will be created if it does not exist.\n", ) @@ -248,87 +246,14 @@ def parse_test_list(full_test_dict, test_string): return test_dict -def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): - """Run the generated Fortran executables - - Args: - build_dir (str): full path to build directory - run_dir (str): full path to run directory - test_dir (str): test directory within the run directory - test_exe (str): test executable to run - args ([str]): arguments for executable - """ - - # move executable to run directory - exe_path = os.path.join(build_dir, _TEST_SUB_DIR, test_dir, test_exe) - copy_file(exe_path, run_dir) - - # run the executable - new_exe_path = os.path.join(run_dir, test_exe) - run_command = [new_exe_path] - run_command.extend(args) - - os.chdir(run_dir) - out = run_cmd_no_fail(" ".join(run_command), combine_output=True) - print(out) - -def make_plotdirs(run_dir, test_dict): - """Create plotting directories if they don't already exist - - Args: - run_dir (str): full path to run directory - test_dict (dict): dictionary of test to run - """ - # make main plot directory - plot_dir = os.path.join(run_dir, 'plots') - if not os.path.isdir(plot_dir): - os.mkdir(plot_dir) - - # make sub-plot directories - for test in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - test_dict.items())): - sub_dir = os.path.join(plot_dir, test) - if not os.path.isdir(sub_dir): - os.mkdir(sub_dir) - -def create_param_file(param_file, run_dir): - """Creates and/or move the default or input parameter file to the run directory - Creates a netcdf file from a cdl file if a cdl file is supplied - - Args: - param_file (str): path to parmaeter file - run_dir (str): full path to run directory - - Raises: - RuntimeError: Supplied parameter file is not netcdf (.cd) or cdl (.cdl) - - Returns: - str: full path to new parameter file name/location - """ - if param_file is None: - print("Using default parameter file.") - param_file = _DEFAULT_CDL_PATH - param_file_update = create_nc_file(param_file, run_dir) - else: - print(f"Using parameter file {param_file}.") - file_suffix = os.path.basename(param_file).split(".")[-1] - if file_suffix == 'cdl': - param_file_update = create_nc_file(param_file, run_dir) - elif file_suffix == "nc": - param_file_update = copy_file(param_file, run_dir) - else: - raise RuntimeError("Must supply parameter file with .cdl or .nc ending.") - - return param_file_update - -def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_dir, +def run_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, make_j, param_file, save_figs, test_dict): """Builds and runs the fates tests Args: clean (bool): whether or not to clean the build directory verbose_make (bool): whether or not to run make with verbose output - build_tests (bool): whether or not to build the exectuables + build (bool): whether or not to build the exectuables run_executables (bool): whether or not to run the executables build_dir (str): build directory run_dir (str): run directory @@ -356,8 +281,8 @@ def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_ param_file = create_param_file(param_file, run_dir) # compile code - if build_tests: - build_unit_tests(build_dir, _TEST_NAME, _CMAKE_BASE_DIR, make_j, clean=clean, + if build: + build_tests(build_dir, _CMAKE_BASE_DIR, make_j, clean=clean, verbose=verbose_make) # run executables for each test in test list @@ -368,6 +293,7 @@ def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_ test_dict.items())).values(): # prepend parameter file (if required) to argument list args = attributes['other_args'] + print(args) if attributes['use_param_file']: args.insert(0, param_file) # run @@ -393,27 +319,80 @@ def run_tests(clean, verbose_make, build_tests, run_executables, build_dir, run_ os.path.join(run_dir_path, 'plots', test)) # show plots plt.show() - -def config_to_dict(config_file): - """Convert a config file to a python dictionary + +def make_plotdirs(run_dir, test_dict): + """Create plotting directories if they don't already exist + + Args: + run_dir (str): full path to run directory + test_dict (dict): dictionary of test to run + """ + # make main plot directory + plot_dir = os.path.join(run_dir, 'plots') + if not os.path.isdir(plot_dir): + os.mkdir(plot_dir) + + # make sub-plot directories + for test in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, + test_dict.items())): + sub_dir = os.path.join(plot_dir, test) + if not os.path.isdir(sub_dir): + os.mkdir(sub_dir) + +def create_param_file(param_file, run_dir): + """Creates and/or move the default or input parameter file to the run directory + Creates a netcdf file from a cdl file if a cdl file is supplied Args: - config_file (str): full path to config file + param_file (str): path to parmaeter file + run_dir (str): full path to run directory + + Raises: + RuntimeError: Supplied parameter file is not netcdf (.cd) or cdl (.cdl) Returns: - dictionary: dictionary of config file + str: full path to new parameter file name/location """ - config = configparser.ConfigParser() - config.read(config_file) - - dictionary = {} - for section in config.sections(): - dictionary[section] = {} - for option in config.options(section): - dictionary[section][option] = config.get(section, option) - - return dictionary - + if param_file is None: + print("Using default parameter file.") + param_file = _DEFAULT_CDL_PATH + param_file_update = create_nc_from_cdl(param_file, run_dir) + else: + print(f"Using parameter file {param_file}.") + file_suffix = os.path.basename(param_file).split(".")[-1] + if file_suffix == 'cdl': + param_file_update = create_nc_from_cdl(param_file, run_dir) + elif file_suffix == "nc": + param_file_update = copy_file(param_file, run_dir) + else: + raise RuntimeError("Must supply parameter file with .cdl or .nc ending.") + + return param_file_update + +def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): + """Run the generated Fortran executables + + Args: + build_dir (str): full path to build directory + run_dir (str): full path to run directory + test_dir (str): test directory within the run directory + test_exe (str): test executable to run + args ([str]): arguments for executable + """ + + # move executable to run directory + exe_path = os.path.join(build_dir, _TEST_SUB_DIR, test_dir, test_exe) + copy_file(exe_path, run_dir) + + # run the executable + new_exe_path = os.path.join(run_dir, test_exe) + run_command = [new_exe_path] + run_command.extend(args) + + os.chdir(run_dir) + out = run_cmd_no_fail(" ".join(run_command), combine_output=True) + print(out) + def main(): """Main script Reads in command-line arguments and then runs the tests. @@ -423,12 +402,12 @@ def main(): args = commandline_args() test_dict = parse_test_list(functional_test_dict, args.test_list) + + #build = not args.skip_build + #run = not args.skip_run_executables - build = not args.skip_build - run = not args.skip_run_executables - - run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, - args.make_j, args.parameter_file, args.save_figs, test_dict) + #run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, + # args.make_j, args.parameter_file, args.save_figs, test_dict) if __name__ == "__main__": main() \ No newline at end of file diff --git a/testing/testing_classes.py b/testing/testing_classes.py new file mode 100644 index 0000000000..ea362e715b --- /dev/null +++ b/testing/testing_classes.py @@ -0,0 +1,22 @@ +from abc import ABC, abstractmethod +from utils import str_to_bool, str_to_list + +class Heuristic(ABC): + """Class for running FATES heuristics""" + + def __init__(self, name, test_dir, test_exe, out_file, use_param_file, other_args): + super().__init__(name, test_dir, test_exe, 'functional') + self.out_file = out_file + self.use_param_file = str_to_bool(use_param_file) + self.other_args = str_to_list(other_args) + + @abstractmethod + def plot_output(self, run_dir:str, out_file:str, save_figs:bool, plot_dir:str): + """plot output for this test + + Args: + run_dir (str): path to run directory + out_file (str): name of output file to read in + save_figs (bool): whether or not to save figs to png + plot_dir (str): where to save figs + """ \ No newline at end of file diff --git a/testing/utils.py b/testing/utils.py index aa6079757d..ea92313b0d 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -3,13 +3,14 @@ import math import os +import configparser from path_utils import add_cime_lib_to_path add_cime_lib_to_path() from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order -def round_up(num, decimals=0): +def round_up(num:float, decimals:int=0) -> float: """Rounds a number up Args: @@ -22,7 +23,7 @@ def round_up(num, decimals=0): multiplier = 10**decimals return math.ceil(num * multiplier)/multiplier -def truncate(num, decimals=0): +def truncate(num:float, decimals:int=0) -> float: """Rounds a number down Args: @@ -35,8 +36,8 @@ def truncate(num, decimals=0): multiplier = 10**decimals return int(num * multiplier)/multiplier -def create_nc_file(cdl_path, run_dir): - """Creates a netcdf file from a cdl file +def create_nc_from_cdl(cdl_path:str, run_dir:str) -> str: + """Creates a netcdf file from a cdl file and return path to new file. Args: cdl_path (str): full path to desired cdl file @@ -55,8 +56,8 @@ def create_nc_file(cdl_path, run_dir): return file_nc_name -def copy_file(file_path, directory): - """Copies a file file to a desired directory +def copy_file(file_path:str, directory) -> str: + """Copies a file file to a desired directory and returns path to file. Args: file_path (str): full path to file @@ -69,26 +70,87 @@ def copy_file(file_path, directory): os.path.abspath(file_path), os.path.abspath(directory) ] - run_cmd_no_fail(" ".join(file_copy_command), combine_output=True) + out = run_cmd_no_fail(" ".join(file_copy_command), combine_output=True) + print(out) return file_basename -def get_color_palette(number): - """Generate a color pallete +def get_color_palette(number:int) -> list: + """_summary_ + Args: - number: number of colors to get - must be <= 20 + number (int): number of colors to get - must be <= 20 + + Raises: + ValueError: number must be less than hard-coded list + Returns: - float: array of colors to use in plotting + list[tuple]: list of colors to use in plotting """ - if number > 20: - raise RuntimeError("get_color_palette: number must be <=20") + + # hard-coded list of colors, can add more here if necessary + all_colors = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120), + (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150), + (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148), + (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199), + (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)] + + if number > len(all_colors): + raise ValueError(f"get_color_palette: number must be <= {len(all_colors)}") + + colors = [(red/255.0, green/255.0, blue/255.0) for red, green, blue in all_colors] - colors = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120), - (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150), - (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148), - (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199), - (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)] + return colors[:number] - colors = [(red/255.0, green/255.0, blue/255.0) for red, green, blue in colors] +def config_to_dict(config_file:str) -> dict: + """Convert a config file to a python dictionary - return colors[:number] + Args: + config_file (str): full path to config file + + Returns: + dictionary: dictionary of config file + """ + config = configparser.ConfigParser() + config.read(config_file) + + dictionary = {} + for section in config.sections(): + dictionary[section] = {} + for option in config.options(section): + dictionary[section][option] = config.get(section, option) + + return dictionary + +def str_to_bool(val:str) -> bool: + """Convert a string representation of truth to True or False. + + Args: + val (str): input string + + Raises: + ValueError: can't figure out what the string should be converted to + + Returns: + bool: True or False + """ + if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): + return True + if val.lower in ('n', 'no', 'f', 'false', 'off', '0'): + return False + raise ValueError(f"invalid truth value {val}") + +def str_to_list(val:str) -> list: + """converts string representation of list to actual list + + Args: + val (str): string representation of list + + Returns: + list: actual list + """ + if val in ('', '[]'): + # empty list + return [] + res = val.strip('][').split(',') + return [n.strip() for n in res] From 2791487e21228ce2caf9a70e2709874320002b84 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 20 Jun 2024 15:31:44 -0600 Subject: [PATCH 3/8] create classes for tests --- testing/build_fortran_tests.py | 2 +- .../allometry/allometry_plotting.py | 187 ------------------ .../allometry/allometry_test.py | 162 ++++++++++++++- .../math_utils/math_plotting.py | 52 ----- .../math_utils/math_utils_test.py | 62 ++++++ testing/functional_tests.cfg | 2 +- ...fates_tests.py => run_functional_tests.py} | 119 ++++++----- testing/testing_classes.py | 19 +- testing/utils.py | 42 +++- 9 files changed, 338 insertions(+), 309 deletions(-) delete mode 100644 testing/functional_testing/allometry/allometry_plotting.py delete mode 100644 testing/functional_testing/math_utils/math_plotting.py create mode 100644 testing/functional_testing/math_utils/math_utils_test.py rename testing/{run_fates_tests.py => run_functional_tests.py} (80%) diff --git a/testing/build_fortran_tests.py b/testing/build_fortran_tests.py index 35f5c51350..509755eb2d 100644 --- a/testing/build_fortran_tests.py +++ b/testing/build_fortran_tests.py @@ -92,7 +92,7 @@ def clean_cmake_files(): or file.startswith(".env_mach_specific") ): os.remove(file) - + def get_extra_cmake_args(build_dir:str, mpilib:str) -> str: """Makes a fake case to grab the required cmake arguments Args: diff --git a/testing/functional_testing/allometry/allometry_plotting.py b/testing/functional_testing/allometry/allometry_plotting.py deleted file mode 100644 index caa6cc1069..0000000000 --- a/testing/functional_testing/allometry/allometry_plotting.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Utility functions for allometry functional unit tests -""" -import os -import math -import pandas as pd -import numpy as np -import xarray as xr -import matplotlib -import matplotlib.pyplot as plt -from utils import get_color_palette, round_up - -def blank_plot(x_max, x_min, y_max, y_min, draw_horizontal_lines=False): - """Generate a blank plot with set attributes - - Args: - x_max (float): maximum x value - x_min (float): minimum x value - y_max (float): maximum y value - y_min (float): minimum y value - draw_horizontal_lines (bool, optional): whether or not to draw horizontal - lines across plot. Defaults to False. - """ - - plt.figure(figsize=(7, 5)) - axis = plt.subplot(111) - axis.spines["top"].set_visible(False) - axis.spines["bottom"].set_visible(False) - axis.spines["right"].set_visible(False) - axis.spines["left"].set_visible(False) - - axis.get_xaxis().tick_bottom() - axis.get_yaxis().tick_left() - - plt.xlim(0.0, x_max) - plt.ylim(0.0, y_max) - - plt.yticks(fontsize=10) - plt.xticks(fontsize=10) - - if draw_horizontal_lines: - inc = (int(y_max) - y_min)/20 - for i in range(0, 20): - plt.plot(range(math.floor(x_min), math.ceil(x_max)), - [0.0 + i*inc] * len(range(math.floor(x_min), math.ceil(x_max))), - "--", lw=0.5, color="black", alpha=0.3) - - plt.tick_params(bottom=False, top=False, left=False, right=False) - - return plt - -def plot_allometry_var(data, varname, units, save_fig, plot_dir=None): - """Plot an allometry variable - - Args: - data (xarray DataArray): the data array of the variable to plot - var (str): variable name (for data structure) - varname (str): variable name for plot labels - units (str): variable units for plot labels - save_fig (bool): whether or not to write out plot - plot_dir (str): if saving figure, where to write to - """ - data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), - 'pft': np.repeat(data.pft, len(data.dbh)), - data.name: data.values.flatten()}) - - max_dbh = data_frame['dbh'].max() - max_var = round_up(data_frame[data.name].max()) - - blank_plot(max_dbh, 0.0, max_var, 0.0, draw_horizontal_lines=True) - - pfts = np.unique(data_frame.pft.values) - colors = get_color_palette(len(pfts)) - for rank, pft in enumerate(pfts): - dat = data_frame[data_frame.pft == pft] - plt.plot(dat.dbh.values, dat[data.name].values, lw=2, color=colors[rank], - label=pft) - - plt.xlabel('DBH (cm)', fontsize=11) - plt.ylabel(f'{varname} ({units})', fontsize=11) - plt.title(f"Simulated {varname} for input parameter file", fontsize=11) - plt.legend(loc='upper left', title='PFT') - - if save_fig: - fig_name = os.path.join(plot_dir, f"allometry_plot_{data.name}.png") - plt.savefig(fig_name) - -def plot_total_biomass(data, save_fig, plot_dir): - """Plot two calculations of total biomass against each other - - Args: - data (xarray DataSet): the allometry dataset - """ - data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), - 'pft': np.repeat(data.pft, len(data.dbh)), - 'total_biomass_parts': data.total_biomass_parts.values.flatten(), - 'total_biomass_tissues': data.total_biomass_tissues.values.flatten()}) - - max_biomass = np.maximum(data_frame['total_biomass_parts'].max(), - data_frame['total_biomass_tissues'].max()) - - blank_plot(max_biomass, 0.0, max_biomass, 0.0, draw_horizontal_lines=False) - - pfts = np.unique(data_frame.pft.values) - colors = get_color_palette(len(pfts)) - for rank, pft in enumerate(pfts): - data = data_frame[data_frame.pft == pft] - plt.scatter(data.total_biomass_parts.values, data.total_biomass_parts.values, - color=colors[rank], label=pft) - - plt.xlabel('Total biomass (kgC) from parts', fontsize=11) - plt.ylabel('Total biomass (kgC) from tissues', fontsize=11) - plt.title("Simulated total biomass for input parameter file", fontsize=11) - plt.legend(loc='upper left', title='PFT') - - if save_fig: - fig_name = os.path.join(plot_dir, "allometry_plot_total_biomass_compare.png") - plt.savefig(fig_name) - -def plot_allometry_dat(run_dir, out_file, save_figs, plot_dir): - """Plots all allometry plots - - Args: - run_dir (str): run directory - out_file (str): output file name - save_figs (bool): whether or not to save the figures - plot_dir (str): plot directory to save the figures to - """ - - # read in allometry data - allometry_dat = xr.open_dataset(os.path.join(run_dir, out_file)) - - plot_dict = { - 'height': { - 'varname': 'height', - 'units': 'm', - }, - 'bagw': { - 'varname': 'aboveground biomass', - 'units': 'kgC', - }, - 'blmax': { - 'varname': 'maximum leaf biomass', - 'units': 'kgC', - }, - 'crown_area': { - 'varname': 'crown area', - 'units': 'm$^2$', - }, - 'sapwood_area': { - 'varname': 'sapwood area', - 'units': 'm$^2$', - }, - 'bsap': { - 'varname': 'sapwood biomass', - 'units': 'kgC', - }, - 'bbgw': { - 'varname': 'belowground biomass', - 'units': 'kgC', - }, - 'fineroot_biomass': { - 'varname': 'fineroot biomass', - 'units': 'kgC', - }, - 'bstore': { - 'varname': 'storage biomass', - 'units': 'kgC', - }, - 'bdead': { - 'varname': 'deadwood biomass', - 'units': 'kgC', - }, - 'total_biomass_parts': { - 'varname': 'total biomass (calculated from parts)', - 'units': 'kgC', - }, - 'total_biomass_tissues': { - 'varname': 'total biomass (calculated from tissues)', - 'units': 'kgC', - }, - - } - for plot, attributes in plot_dict.items(): - plot_allometry_var(allometry_dat[plot], attributes['varname'], - attributes['units'], save_figs, plot_dir) - - plot_total_biomass(allometry_dat, save_figs, plot_dir) diff --git a/testing/functional_testing/allometry/allometry_test.py b/testing/functional_testing/allometry/allometry_test.py index fe800b91dd..231eb3e136 100644 --- a/testing/functional_testing/allometry/allometry_test.py +++ b/testing/functional_testing/allometry/allometry_test.py @@ -1,5 +1,159 @@ -from testing_classes import FunctionalTest +import os +import math +import xarray as xr +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +from utils import round_up, get_color_palette, blank_plot +from testing_classes import Heuristic -class AllometryTest(FunctionalTest): - def __init__() - \ No newline at end of file +class AllometryTest(Heuristic): + + name = 'allometry' + + def __init__(self, test_dict): + super().__init__(AllometryTest.name, test_dict['test_dir'], + test_dict['test_exe'], test_dict['out_file'], + test_dict['use_param_file'], + test_dict['other_args']) + + def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): + """Plots all allometry plots + + Args: + run_dir (str): run directory + out_file (str): output file name + save_figs (bool): whether or not to save the figures + plot_dir (str): plot directory to save the figures to + """ + + # read in allometry data + allometry_dat = xr.open_dataset(os.path.join(run_dir, self.out_file)) + + plot_dict = { + 'height': { + 'varname': 'height', + 'units': 'm', + }, + 'bagw': { + 'varname': 'aboveground biomass', + 'units': 'kgC', + }, + 'blmax': { + 'varname': 'maximum leaf biomass', + 'units': 'kgC', + }, + 'crown_area': { + 'varname': 'crown area', + 'units': 'm$^2$', + }, + 'sapwood_area': { + 'varname': 'sapwood area', + 'units': 'm$^2$', + }, + 'bsap': { + 'varname': 'sapwood biomass', + 'units': 'kgC', + }, + 'bbgw': { + 'varname': 'belowground biomass', + 'units': 'kgC', + }, + 'fineroot_biomass': { + 'varname': 'fineroot biomass', + 'units': 'kgC', + }, + 'bstore': { + 'varname': 'storage biomass', + 'units': 'kgC', + }, + 'bdead': { + 'varname': 'deadwood biomass', + 'units': 'kgC', + }, + 'total_biomass_parts': { + 'varname': 'total biomass (calculated from parts)', + 'units': 'kgC', + }, + 'total_biomass_tissues': { + 'varname': 'total biomass (calculated from tissues)', + 'units': 'kgC', + }, + + } + for plot, attributes in plot_dict.items(): + self.plot_allometry_var(allometry_dat[plot], attributes['varname'], + attributes['units'], save_figs, plot_dir) + + self.plot_total_biomass(allometry_dat, save_figs, plot_dir) + + @staticmethod + def plot_allometry_var(data: xr.Dataset, varname: str, units: str, + save_fig: bool, plot_dir :str=None): + """Plot an allometry variable + + Args: + data (xarray DataArray): the data array of the variable to plot + var (str): variable name (for data structure) + varname (str): variable name for plot labels + units (str): variable units for plot labels + save_fig (bool): whether or not to write out plot + plot_dir (str): if saving figure, where to write to + """ + data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), + 'pft': np.repeat(data.pft, len(data.dbh)), + data.name: data.values.flatten()}) + + max_dbh = data_frame['dbh'].max() + max_var = round_up(data_frame[data.name].max()) + + blank_plot(max_dbh, 0.0, max_var, 0.0, draw_horizontal_lines=True) + + pfts = np.unique(data_frame.pft.values) + colors = get_color_palette(len(pfts)) + for rank, pft in enumerate(pfts): + dat = data_frame[data_frame.pft == pft] + plt.plot(dat.dbh.values, dat[data.name].values, lw=2, color=colors[rank], + label=pft) + + plt.xlabel('DBH (cm)', fontsize=11) + plt.ylabel(f'{varname} ({units})', fontsize=11) + plt.title(f"Simulated {varname} for input parameter file", fontsize=11) + plt.legend(loc='upper left', title='PFT') + + if save_fig: + fig_name = os.path.join(plot_dir, f"allometry_plot_{data.name}.png") + plt.savefig(fig_name) + + @staticmethod + def plot_total_biomass(data: xr.Dataset, save_fig: bool, plot_dir: str): + """Plot two calculations of total biomass against each other + + Args: + data (xarray DataSet): the allometry dataset + """ + data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), + 'pft': np.repeat(data.pft, len(data.dbh)), + 'total_biomass_parts': data.total_biomass_parts.values.flatten(), + 'total_biomass_tissues': data.total_biomass_tissues.values.flatten()}) + + max_biomass = np.maximum(data_frame['total_biomass_parts'].max(), + data_frame['total_biomass_tissues'].max()) + + blank_plot(max_biomass, 0.0, max_biomass, 0.0, draw_horizontal_lines=False) + + pfts = np.unique(data_frame.pft.values) + colors = get_color_palette(len(pfts)) + for rank, pft in enumerate(pfts): + data = data_frame[data_frame.pft == pft] + plt.scatter(data.total_biomass_parts.values, data.total_biomass_parts.values, + color=colors[rank], label=pft) + + plt.xlabel('Total biomass (kgC) from parts', fontsize=11) + plt.ylabel('Total biomass (kgC) from tissues', fontsize=11) + plt.title("Simulated total biomass for input parameter file", fontsize=11) + plt.legend(loc='upper left', title='PFT') + + if save_fig: + fig_name = os.path.join(plot_dir, "allometry_plot_total_biomass_compare.png") + plt.savefig(fig_name) diff --git a/testing/functional_testing/math_utils/math_plotting.py b/testing/functional_testing/math_utils/math_plotting.py deleted file mode 100644 index 4e386dbe93..0000000000 --- a/testing/functional_testing/math_utils/math_plotting.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Utility functions for allometry functional unit tests -""" -import os -import math -import xarray as xr -import numpy as np -import matplotlib.pyplot as plt - -from utils import get_color_palette - -def plot_quadratic_dat(run_dir, out_file, save_figs, plot_dir): - """Reads in and plots quadratic formula test output - - Args: - run_dir (str): run directory - out_file (str): output file - save_figs (bool): whether or not to save the figures - plot_dir (str): plot directory - """ - - # read in quadratic data - quadratic_dat = xr.open_dataset(os.path.join(run_dir, out_file)) - - # plot output - plot_quad_and_roots(quadratic_dat.a.values, quadratic_dat.b.values, - quadratic_dat.c.values, quadratic_dat.root1.values, - quadratic_dat.root2.values) - if save_figs: - fig_name = os.path.join(plot_dir, "quadratic_test.png") - plt.savefig(fig_name) - -def plot_quad_and_roots(a_coeff, b_coeff, c_coeff, root1, root2): - """Plots a set of quadratic formulas (ax**2 + bx + c) and their two roots - - Args: - a_coeff (float array): set of a coefficients - b_coeff (float array): set of b coefficients - c_coeff (float array): set of b coefficients - root1 (float array): set of first real roots - root2 (float array): set of second real roots - """ - num_equations = len(a_coeff) - - plt.figure(figsize=(7, 5)) - x_vals = np.linspace(-10.0, 10.0, num=20) - - colors = get_color_palette(num_equations) - for i in range(num_equations): - y_vals = a_coeff[i]*x_vals**2 + b_coeff[i]*x_vals + c_coeff[i] - plt.plot(x_vals, y_vals, lw=2, color=colors[i]) - plt.scatter(root1[i], root2[i], color=colors[i], s=50) - plt.axhline(y=0.0, color='k', linestyle='dotted') diff --git a/testing/functional_testing/math_utils/math_utils_test.py b/testing/functional_testing/math_utils/math_utils_test.py new file mode 100644 index 0000000000..094d222aeb --- /dev/null +++ b/testing/functional_testing/math_utils/math_utils_test.py @@ -0,0 +1,62 @@ +import os +import math +import xarray as xr +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +from utils import get_color_palette +from testing_classes import Heuristic + +class QuadraticTest(Heuristic): + + name = 'quadratic' + + def __init__(self, test_dict): + super().__init__(QuadraticTest.name, test_dict['test_dir'], + test_dict['test_exe'], test_dict['out_file'], + test_dict['use_param_file'], + test_dict['other_args']) + + def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): + """Reads in and plots quadratic formula test output + + Args: + run_dir (str): run directory + out_file (str): output file + save_figs (bool): whether or not to save the figures + plot_dir (str): plot directory + """ + + # read in quadratic data + quadratic_dat = xr.open_dataset(os.path.join(run_dir, self.out_file)) + + # plot output + self.plot_quad_and_roots(quadratic_dat.a.values, quadratic_dat.b.values, + quadratic_dat.c.values, quadratic_dat.root1.values, + quadratic_dat.root2.values) + if save_figs: + fig_name = os.path.join(plot_dir, "quadratic_test.png") + plt.savefig(fig_name) + + @staticmethod + def plot_quad_and_roots(a_coeff, b_coeff, c_coeff, root1, root2): + """Plots a set of quadratic formulas (ax**2 + bx + c) and their two roots + + Args: + a_coeff (float array): set of a coefficients + b_coeff (float array): set of b coefficients + c_coeff (float array): set of b coefficients + root1 (float array): set of first real roots + root2 (float array): set of second real roots + """ + num_equations = len(a_coeff) + + plt.figure(figsize=(7, 5)) + x_vals = np.linspace(-10.0, 10.0, num=20) + + colors = get_color_palette(num_equations) + for i in range(num_equations): + y_vals = a_coeff[i]*x_vals**2 + b_coeff[i]*x_vals + c_coeff[i] + plt.plot(x_vals, y_vals, lw=2, color=colors[i]) + plt.scatter(root1[i], root2[i], color=colors[i], s=50) + plt.axhline(y=0.0, color='k', linestyle='dotted') \ No newline at end of file diff --git a/testing/functional_tests.cfg b/testing/functional_tests.cfg index 3c16e8cb1c..f6c9c3f69c 100644 --- a/testing/functional_tests.cfg +++ b/testing/functional_tests.cfg @@ -10,4 +10,4 @@ test_dir = fates_math_ftest test_exe = FATES_math_exe out_file = quad_out.nc use_param_file = False -other_args = [] \ No newline at end of file +other_args = [] diff --git a/testing/run_fates_tests.py b/testing/run_functional_tests.py similarity index 80% rename from testing/run_fates_tests.py rename to testing/run_functional_tests.py index 9d7cc28801..833df5d487 100755 --- a/testing/run_fates_tests.py +++ b/testing/run_functional_tests.py @@ -19,8 +19,8 @@ You must also have a .cime folder in your home directory which specifies machine configurations for CIME. -This script builds and runs various FATES unit and functional tests, and plots any -relevant output from those tests. +This script builds and runs FATES functional tests, and plots any relevant output from +those tests. You can supply your own parameter file (either a .cdl or a .nc file), or if you do not specify anything, the script will use the default FATES parameter cdl file. @@ -29,11 +29,15 @@ import os import argparse import matplotlib.pyplot as plt + from build_fortran_tests import build_tests, build_exists from path_utils import add_cime_lib_to_path -from utils import copy_file, create_nc_from_cdl, config_to_dict, str_to_list -from functional_testing.allometry.allometry_plotting import plot_allometry_dat -from functional_testing.math_utils.math_plotting import plot_quadratic_dat +from utils import copy_file, create_nc_from_cdl, config_to_dict + +# add testing subclasses here +from testing_classes import Heuristic +from functional_testing.allometry.allometry_test import AllometryTest +from functional_testing.math_utils.math_utils_test import QuadraticTest add_cime_lib_to_path() @@ -49,7 +53,7 @@ def commandline_args(): """Parse and return command-line arguments""" description = """ - Driver for running FATES unit and functional tests + Driver for running FATES functional tests Typical usage: @@ -159,6 +163,13 @@ def check_arg_validity(args): """ # check to make sure parameter file exists and is one of the correct forms check_param_file(args.parameter_file) + + # make sure relevant output files exist: + if args.skip_run_executables: + # if you skip the run we assume you want to skip the build + print("--skip-run specified, assuming --skip-build") + args.skip_build = True + check_out_files(args.run_dir, args.test_dict) # make sure build directory exists if args.skip_build: @@ -167,10 +178,6 @@ def check_arg_validity(args): "Re-run script without --skip-build") check_build_dir(args.build_dir, args.test_dict) - # make sure relevant output files exist: - if args.skip_run_executables: - check_out_files(args.run_dir, args.test_dict) - def check_param_file(param_file): """Checks to see if param_file exists and is of the correct form (.nc or .cdl) @@ -201,7 +208,7 @@ def check_build_dir(build_dir, test_dict): if not build_exists(build_dir, attributes['test_dir'], attributes['test_exe']): raise argparse.ArgumentError(None, "Build directory or executable does not exist.\n" "Re-run script without --skip-build.") - + def check_out_files(run_dir, test_dict): """Checks to see that required output files are present in the run directory @@ -285,40 +292,40 @@ def run_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, build_tests(build_dir, _CMAKE_BASE_DIR, make_j, clean=clean, verbose=verbose_make) - # run executables for each test in test list - if run_executables: - print("Running executables") - # we don't run executables for only pfunit tests - for attributes in dict(filter(lambda pair: pair[1]['test_exe'] is not None, - test_dict.items())).values(): - # prepend parameter file (if required) to argument list - args = attributes['other_args'] - print(args) - if attributes['use_param_file']: - args.insert(0, param_file) - # run - run_fortran_exectuables(build_dir_path, attributes['test_dir'], - attributes['test_exe'], run_dir_path, args) + # # run executables for each test in test list + # if run_executables: + # print("Running executables") + # # we don't run executables for only pfunit tests + # for attributes in dict(filter(lambda pair: pair[1]['test_exe'] is not None, + # test_dict.items())).values(): + # # prepend parameter file (if required) to argument list + # args = attributes['other_args'] + # print(args) + # if attributes['use_param_file']: + # args.insert(0, param_file) + # # run + # run_fortran_exectuables(build_dir_path, attributes['test_dir'], + # attributes['test_exe'], run_dir_path, args) - # run unit tests - for test, attributes in dict(filter(lambda pair: pair[1]['has_unit_test'], - test_dict.items())).items(): - print(f"Running unit tests for {test}.") + # # run unit tests + # for test, attributes in dict(filter(lambda pair: pair[1]['has_unit_test'], + # test_dict.items())).items(): + # print(f"Running unit tests for {test}.") - test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) - ctest_command = ["ctest", "--output-on-failure"] - output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, - combine_output=True) - print(output) - - # plot output for relevant tests - for test, attributes in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - test_dict.items())).items(): - attributes['plotting_function'](run_dir_path, - attributes['out_file'], save_figs, - os.path.join(run_dir_path, 'plots', test)) - # show plots - plt.show() + # test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) + # ctest_command = ["ctest", "--output-on-failure"] + # output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, + # combine_output=True) + # print(output) + + # # plot output for relevant tests + # for test, attributes in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, + # test_dict.items())).items(): + # attributes['plotting_function'](run_dir_path, + # attributes['out_file'], save_figs, + # os.path.join(run_dir_path, 'plots', test)) + # # show plots + # plt.show() def make_plotdirs(run_dir, test_dict): """Create plotting directories if they don't already exist @@ -398,16 +405,24 @@ def main(): Reads in command-line arguments and then runs the tests. """ - functional_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) + full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) + subclasses = Heuristic.__subclasses__() args = commandline_args() - test_dict = parse_test_list(functional_test_dict, args.test_list) + config_dict = parse_test_list(full_test_dict, args.test_list) - #build = not args.skip_build - #run = not args.skip_run_executables - - #run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, - # args.make_j, args.parameter_file, args.save_figs, test_dict) - + ## for now just turn into a dictionary? + test_dict = {} + for name in config_dict.keys(): + my_class = list(filter(lambda subclass: subclass.name == name, subclasses))[0](config_dict[name]) + test_dict[name] = my_class.to_dict() + print(test_dict) + + build = not args.skip_build + run = not args.skip_run_executables + + # run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, + # args.make_j, args.parameter_file, args.save_figs, test_dict) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/testing/testing_classes.py b/testing/testing_classes.py index ea362e715b..35e2b93eb2 100644 --- a/testing/testing_classes.py +++ b/testing/testing_classes.py @@ -4,19 +4,16 @@ class Heuristic(ABC): """Class for running FATES heuristics""" - def __init__(self, name, test_dir, test_exe, out_file, use_param_file, other_args): - super().__init__(name, test_dir, test_exe, 'functional') + def __init__(self, name:str, test_dir:str, test_exe:str, out_file:str, + use_param_file:str, other_args:str): + self.name = name + self.test_dir = test_dir + self.text_exe = test_exe self.out_file = out_file self.use_param_file = str_to_bool(use_param_file) self.other_args = str_to_list(other_args) @abstractmethod - def plot_output(self, run_dir:str, out_file:str, save_figs:bool, plot_dir:str): - """plot output for this test - - Args: - run_dir (str): path to run directory - out_file (str): name of output file to read in - save_figs (bool): whether or not to save figs to png - plot_dir (str): where to save figs - """ \ No newline at end of file + def plot_output(self, run_dir:str, save_figs:bool, plot_dir:str): + pass + \ No newline at end of file diff --git a/testing/utils.py b/testing/utils.py index ea92313b0d..7ddf597fb4 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -4,6 +4,7 @@ import math import os import configparser +import matplotlib.pyplot as plt from path_utils import add_cime_lib_to_path add_cime_lib_to_path() @@ -136,7 +137,7 @@ def str_to_bool(val:str) -> bool: """ if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): return True - if val.lower in ('n', 'no', 'f', 'false', 'off', '0'): + if val.lower() in ('n', 'no', 'f', 'false', 'off', '0'): return False raise ValueError(f"invalid truth value {val}") @@ -154,3 +155,42 @@ def str_to_list(val:str) -> list: return [] res = val.strip('][').split(',') return [n.strip() for n in res] + +def blank_plot(x_max: float, x_min: float, y_max: float, y_min: float, + draw_horizontal_lines :bool=False): + """Generate a blank plot with set attributes + + Args: + x_max (float): maximum x value + x_min (float): minimum x value + y_max (float): maximum y value + y_min (float): minimum y value + draw_horizontal_lines (bool, optional): whether or not to draw horizontal + lines across plot. Defaults to False. + """ + + plt.figure(figsize=(7, 5)) + axis = plt.subplot(111) + axis.spines["top"].set_visible(False) + axis.spines["bottom"].set_visible(False) + axis.spines["right"].set_visible(False) + axis.spines["left"].set_visible(False) + + axis.get_xaxis().tick_bottom() + axis.get_yaxis().tick_left() + + plt.xlim(0.0, x_max) + plt.ylim(0.0, y_max) + + plt.yticks(fontsize=10) + plt.xticks(fontsize=10) + + if draw_horizontal_lines: + inc = (int(y_max) - y_min)/20 + for i in range(0, 20): + plt.plot(range(math.floor(x_min), math.ceil(x_max)), + [0.0 + i*inc] * len(range(math.floor(x_min), math.ceil(x_max))), + "--", lw=0.5, color="black", alpha=0.3) + + plt.tick_params(bottom=False, top=False, left=False, right=False) + From 2c387036cdf155a6143e81697c567799a81f213b Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Sep 2024 11:38:59 -0600 Subject: [PATCH 4/8] testing updates --- .../allometry/allometry_test.py | 1 + .../math_utils/math_utils_test.py | 1 + testing/run_functional_tests.py | 106 ++++--------- testing/run_unit_tests.py | 142 ++++++++++++++++++ testing/testing_classes.py | 5 +- testing/unit_tests.cfg | 2 + testing/utils.py | 29 ++++ 7 files changed, 209 insertions(+), 77 deletions(-) create mode 100755 testing/run_unit_tests.py create mode 100644 testing/unit_tests.cfg diff --git a/testing/functional_testing/allometry/allometry_test.py b/testing/functional_testing/allometry/allometry_test.py index 231eb3e136..7896b6678c 100644 --- a/testing/functional_testing/allometry/allometry_test.py +++ b/testing/functional_testing/allometry/allometry_test.py @@ -16,6 +16,7 @@ def __init__(self, test_dict): test_dict['test_exe'], test_dict['out_file'], test_dict['use_param_file'], test_dict['other_args']) + self.plot = True def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): """Plots all allometry plots diff --git a/testing/functional_testing/math_utils/math_utils_test.py b/testing/functional_testing/math_utils/math_utils_test.py index 094d222aeb..fa978118a6 100644 --- a/testing/functional_testing/math_utils/math_utils_test.py +++ b/testing/functional_testing/math_utils/math_utils_test.py @@ -16,6 +16,7 @@ def __init__(self, test_dict): test_dict['test_exe'], test_dict['out_file'], test_dict['use_param_file'], test_dict['other_args']) + self.plot = True def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): """Reads in and plots quadratic formula test output diff --git a/testing/run_functional_tests.py b/testing/run_functional_tests.py index 833df5d487..f0c01762ee 100755 --- a/testing/run_functional_tests.py +++ b/testing/run_functional_tests.py @@ -32,7 +32,7 @@ from build_fortran_tests import build_tests, build_exists from path_utils import add_cime_lib_to_path -from utils import copy_file, create_nc_from_cdl, config_to_dict +from utils import copy_file, create_nc_from_cdl, config_to_dict, parse_test_list # add testing subclasses here from testing_classes import Heuristic @@ -57,7 +57,7 @@ def commandline_args(): Typical usage: - ./run_fates_tests -t allometry + ./run_fates_functional_tests -t allometry """ parser = argparse.ArgumentParser( @@ -225,37 +225,9 @@ def check_out_files(run_dir, test_dict): raise argparse.ArgumentError(None, f"Required file for {test} test does not exist.\n" "Re-run script without --skip-run.") -def parse_test_list(full_test_dict, test_string): - """Parses the input test list and checks for errors - - Args: - test (str): user-supplied comma-separated list of test names - - Returns: - dictionary: filtered dictionary of tests to run - - Raises: - RuntimeError: Invalid test name supplied - """ - valid_test_names = full_test_dict.keys() - - if test_string != "all": - test_list = test_string.split(',') - for test in test_list: - if test not in valid_test_names: - raise argparse.ArgumentTypeError("Invalid test supplied, \n" - "must supply one of:\n" - f"{', '.join(valid_test_names)}\n" - "or do not supply a test name to run all tests.") - test_dict = {key: full_test_dict[key] for key in test_list} - else: - test_dict = full_test_dict - - return test_dict - -def run_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, +def run_functional_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, make_j, param_file, save_figs, test_dict): - """Builds and runs the fates tests + """Builds and runs the fates functional tests Args: clean (bool): whether or not to clean the build directory @@ -267,7 +239,7 @@ def run_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, make_j (int): number of processors for the build param_file (str): input FATES parameter file save_figs (bool): whether or not to write figures to file - test_dict (dict): dictionary of tests to run + test_dict (dict): dictionary of test classes to run """ # absolute path to desired build directory @@ -292,40 +264,25 @@ def run_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, build_tests(build_dir, _CMAKE_BASE_DIR, make_j, clean=clean, verbose=verbose_make) - # # run executables for each test in test list - # if run_executables: - # print("Running executables") - # # we don't run executables for only pfunit tests - # for attributes in dict(filter(lambda pair: pair[1]['test_exe'] is not None, - # test_dict.items())).values(): - # # prepend parameter file (if required) to argument list - # args = attributes['other_args'] - # print(args) - # if attributes['use_param_file']: - # args.insert(0, param_file) - # # run - # run_fortran_exectuables(build_dir_path, attributes['test_dir'], - # attributes['test_exe'], run_dir_path, args) + # run executables for each test in test list + if run_executables: + print("Running executables") + for _, test in test_dict.items(): + # prepend parameter file (if required) to argument list + args = test.other_args + if test.use_param_file: + args.insert(0, param_file) + # run + run_fortran_exectuables(build_dir_path, test.test_dir, + test.test_exe, run_dir_path, args) - # # run unit tests - # for test, attributes in dict(filter(lambda pair: pair[1]['has_unit_test'], - # test_dict.items())).items(): - # print(f"Running unit tests for {test}.") - - # test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) - # ctest_command = ["ctest", "--output-on-failure"] - # output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, - # combine_output=True) - # print(output) - - # # plot output for relevant tests - # for test, attributes in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - # test_dict.items())).items(): - # attributes['plotting_function'](run_dir_path, - # attributes['out_file'], save_figs, - # os.path.join(run_dir_path, 'plots', test)) - # # show plots - # plt.show() + # plot output for relevant tests + for name, test in dict(filter(lambda pair: pair[1].plot, + test_dict.items())).items(): + test.plot_output(run_dir_path, save_figs, + os.path.join(run_dir_path, 'plots', name)) + # show plots + plt.show() def make_plotdirs(run_dir, test_dict): """Create plotting directories if they don't already exist @@ -338,10 +295,9 @@ def make_plotdirs(run_dir, test_dict): plot_dir = os.path.join(run_dir, 'plots') if not os.path.isdir(plot_dir): os.mkdir(plot_dir) - + # make sub-plot directories - for test in dict(filter(lambda pair: pair[1]['plotting_function'] is not None, - test_dict.items())): + for test in dict(filter(lambda pair: pair[1].plot, test_dict.items())): sub_dir = os.path.join(plot_dir, test) if not os.path.isdir(sub_dir): os.mkdir(sub_dir) @@ -411,18 +367,18 @@ def main(): args = commandline_args() config_dict = parse_test_list(full_test_dict, args.test_list) - ## for now just turn into a dictionary? + # for now just turn into a dictionary? test_dict = {} for name in config_dict.keys(): - my_class = list(filter(lambda subclass: subclass.name == name, subclasses))[0](config_dict[name]) - test_dict[name] = my_class.to_dict() - print(test_dict) + test_class = list(filter(lambda subclass: subclass.name == name, subclasses))[0](config_dict[name]) + test_dict[name] = test_class build = not args.skip_build run = not args.skip_run_executables - # run_tests(args.clean, args.verbose_make, build, run, args.build_dir, args.run_dir, - # args.make_j, args.parameter_file, args.save_figs, test_dict) + run_functional_tests(args.clean, args.verbose_make, build, run, args.build_dir, + args.run_dir, args.make_j, args.parameter_file, args.save_figs, + test_dict) if __name__ == "__main__": main() diff --git a/testing/run_unit_tests.py b/testing/run_unit_tests.py new file mode 100755 index 0000000000..988baa1f37 --- /dev/null +++ b/testing/run_unit_tests.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +""" +|------------------------------------------------------------------| +|--------------------- Instructions -----------------------------| +|------------------------------------------------------------------| +To run this script the following python packages are required: + - numpy + - xarray + - matplotlib + - pandas + +Though this script does not require any host land model code, it does require some CIME +and shr code, so you should still get these repositories as you normally would +(i.e., manage_externals, etc.) + +Additionally, this requires netcdf and netcdff as well as a fortran compiler. + +You must also have a .cime folder in your home directory which specifies machine +configurations for CIME. + +This script builds and runs FATES units tests. + +""" +import os +import argparse + +from build_fortran_tests import build_tests +from path_utils import add_cime_lib_to_path +from utils import config_to_dict, parse_test_list + +add_cime_lib_to_path() + +from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order + +# constants for this script +_CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") +_DEFAULT_CONFIG_FILE = "unit_tests.cfg" +_TEST_SUB_DIR = "testing" + +def commandline_args(): + """Parse and return command-line arguments""" + + description = """ + Driver for running FATES unit tests + + Typical usage: + + ./run_fates_unit_tests -t fire_weather + + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument( + "-b", + "--build-dir", + type=str, + default=os.path.join(_CMAKE_BASE_DIR, "_build"), + help="Directory where tests are built.\n" + "Will be created if it does not exist.\n", + ) + + parser.add_argument( + "--make-j", + type=int, + default=8, + help="Number of processes to use for build.", + ) + + parser.add_argument( + "-c", + "--clean", + action="store_true", + help="Clean build directory before building.\n" + "Removes CMake cache and runs 'make clean'.\n", + ) + + parser.add_argument( + "--verbose-make", + action="store_true", + help="Run make with verbose output." + ) + + parser.add_argument( + "-t", + "--test-list", + action="store", + dest="test_list", + type=str, + default="all", + help="Test(s) to run. Comma-separated list of test names, or 'all'\n" + "for all tests. If not supplied, will run all tests." + ) + + args = parser.parse_args() + + return args + +def run_unit_tests(clean, verbose_make, build_dir, make_j, test_dict): + """Builds and runs the fates unit tests + + Args: + clean (bool): whether or not to clean the build directory + verbose_make (bool): whether or not to run make with verbose output + build_dir (str): build directory + make_j (int): number of processors for the build + test_dict (dict): dictionary of test classes to run + """ + + # absolute path to desired build directory + build_dir_path = os.path.abspath(build_dir) + + # compile code + build_tests(build_dir_path, _CMAKE_BASE_DIR, make_j, clean=clean, + verbose=verbose_make) + + # run unit tests + print(f"Running unit tests...") + for _, attributes in test_dict.items(): + + test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) + ctest_command = ["ctest", "--output-on-failure"] + output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, + combine_output=True) + print(output) + +def main(): + """Main script + Reads in command-line arguments and then runs the tests. + """ + + full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) + + args = commandline_args() + test_dict = parse_test_list(full_test_dict, args.test_list) + + run_unit_tests(args.clean, args.verbose_make, args.build_dir, args.make_j, test_dict) + +if __name__ == "__main__": + main() diff --git a/testing/testing_classes.py b/testing/testing_classes.py index 35e2b93eb2..3092df511a 100644 --- a/testing/testing_classes.py +++ b/testing/testing_classes.py @@ -8,11 +8,12 @@ def __init__(self, name:str, test_dir:str, test_exe:str, out_file:str, use_param_file:str, other_args:str): self.name = name self.test_dir = test_dir - self.text_exe = test_exe + self.test_exe = test_exe self.out_file = out_file self.use_param_file = str_to_bool(use_param_file) self.other_args = str_to_list(other_args) - + self.plot = False + @abstractmethod def plot_output(self, run_dir:str, save_figs:bool, plot_dir:str): pass diff --git a/testing/unit_tests.cfg b/testing/unit_tests.cfg new file mode 100644 index 0000000000..3e5ee536f4 --- /dev/null +++ b/testing/unit_tests.cfg @@ -0,0 +1,2 @@ +[fire_weather] +test_dir = fates_fire_weather_utest diff --git a/testing/utils.py b/testing/utils.py index 7ddf597fb4..d6201bdceb 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -4,6 +4,7 @@ import math import os import configparser +import argparse import matplotlib.pyplot as plt from path_utils import add_cime_lib_to_path @@ -123,6 +124,34 @@ def config_to_dict(config_file:str) -> dict: return dictionary +def parse_test_list(full_test_dict, test_string): + """Parses the input test list and checks for errors + + Args: + test (str): user-supplied comma-separated list of test names + + Returns: + dictionary: filtered dictionary of tests to run + + Raises: + RuntimeError: Invalid test name supplied + """ + valid_test_names = full_test_dict.keys() + + if test_string != "all": + test_list = test_string.split(',') + for test in test_list: + if test not in valid_test_names: + raise argparse.ArgumentTypeError("Invalid test supplied, \n" + "must supply one of:\n" + f"{', '.join(valid_test_names)}\n" + "or do not supply a test name to run all tests.") + test_dict = {key: full_test_dict[key] for key in test_list} + else: + test_dict = full_test_dict + + return test_dict + def str_to_bool(val:str) -> bool: """Convert a string representation of truth to True or False. From 10655d997cbda225116d49ae436fde31eee6e440 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Sep 2024 13:21:01 -0600 Subject: [PATCH 5/8] add testing README --- testing/README.testing | 0 testing/run_unit_tests.py | 5 ----- 2 files changed, 5 deletions(-) create mode 100644 testing/README.testing diff --git a/testing/README.testing b/testing/README.testing new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/run_unit_tests.py b/testing/run_unit_tests.py index 988baa1f37..2656bda0e5 100755 --- a/testing/run_unit_tests.py +++ b/testing/run_unit_tests.py @@ -4,11 +4,6 @@ |------------------------------------------------------------------| |--------------------- Instructions -----------------------------| |------------------------------------------------------------------| -To run this script the following python packages are required: - - numpy - - xarray - - matplotlib - - pandas Though this script does not require any host land model code, it does require some CIME and shr code, so you should still get these repositories as you normally would From 41d411e83f77906669a8b1ce7aa301acf7ce4e78 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Sep 2024 14:17:34 -0600 Subject: [PATCH 6/8] update information about how to run --- testing/README.testing | 0 testing/README.testing.md | 92 +++++++++++++ testing/cime_setup.md | 122 ++++++++++++++++++ ...testing_classes.py => functional_class.py} | 4 +- .../allometry/allometry_test.py | 4 +- .../math_utils/math_utils_test.py | 4 +- testing/run_functional_tests.py | 4 +- testing/testing.yml | 14 ++ 8 files changed, 236 insertions(+), 8 deletions(-) delete mode 100644 testing/README.testing create mode 100644 testing/README.testing.md create mode 100644 testing/cime_setup.md rename testing/{testing_classes.py => functional_class.py} (88%) create mode 100644 testing/testing.yml diff --git a/testing/README.testing b/testing/README.testing deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/testing/README.testing.md b/testing/README.testing.md new file mode 100644 index 0000000000..5275ec7816 --- /dev/null +++ b/testing/README.testing.md @@ -0,0 +1,92 @@ +# FATES Testing + +These scripts set up, build, and run FATES functional tests and unit tests. + +By "functional" test, we mean a standalone Fortran program that runs pieces of the FATES +production code, potentially outputs results (i.e. to a netcdf file), and potentially +plots or runs some other test on the output in python. These tests do not necessarily +have a pass/fail status, but are meant to be more hands-on for the user. + +Unit tests do have a pass/fail outcome, and are written as such. We accommodate both Ctest +and pfunit tests. + +## How to run + +To run the testing scripts, `run_functional_tests.py` or `run_unit_tests.py`, you will +need a few python packages. You can create a conda environment with these packages +using the `testing.yml` file: `conda env create --file=testing.yml` + +Though these tests do not need host land model code, you will need the `cime` and `shr` +repositories, as well as a machine configuration file. If you are already set up to run +FATES on a machine (e.g. derecho at NCAR), you should be able to run these scripts out +of the box on that machine. + +Additionally, these tests require netcdf and netcdff, as well as a fortran compiler (e.g. gnu), +esmf, and pfunit. See `cime_setup.md` for tips on how to do this. + +Once you are set up, you should be able to just run the scripts. For the functional test +script, you can point to your own parameter file (cdl or nc; `./run_functional_tests -f my_param_file.nc`). +If none is supplied the script will use the default cdl file in `parameter_files`. + +You can run an individual set of tests by passing the script a comma-separated list of +test names. See the `functional_tests.cfg` or `unit_tests.cfg` for the test names. If you +do not supply a list, the script will run all tests. + +## How to create new tests + +First, determine if you are going to create a functional or unit test. Remember, +unit tests must have a pass/fail outcome. These are best for testing edgecases, error +handling, or where we know exactly what the result should be from a method. + +First, add your test to either the `functional_tests.cfg` or `unit_tests.cfg` config file, depending on the test you want to create. + +### Config file information + +The `test_dir` is where cmake will place relevant libraries created from your test. The convention is to call this directory `fates_{testname}_ftest` for functional tests and `fates_{testname}_utest` for unit tests. + +The `test_exe` (only applicable for functional tests) is the executable the script will +create based on your test program. The convention is to call it `FATES_{testname}_exe`. + +The `out_file` (only applicable for functional tests) is the output file name that your test +may or may not create. Set the value to `None` if your test does not create one. + +Set `use_param_file` to `True` if your test uses the FATES parameter file, and `False` +otherwise. This is only applicable for functional tests. + +Add any other arguments your test needs in the `other_args` list. This is only applicable for functional tests. + +### Cmake setup + +Under the `testing/functional_testing` or `testing/unit_testing` directory, depending +on your test type, create a new directory for your test, e.g. "my_new_test". + +In the file `testing/CMakeLists.txt` add your test directory to the list of tests, e.g.: + +`add_subdirectory(functional_testing/my_new_test fates_new_test_ftest)` + +The first argument must match your directory name, and the second must match the `test_dir` value you set up in the config file above. + +Inside your new testing directory create a new `CMakeLists.txt` file to tell cmake +how to build and compile your program. It may be easiest to copy an existing one +from another similar test and then update the relevant information. + +Importantly, the sources must be set to tell the program which file(s) contain your +test program. Additionally, for functional tests, the executable name must match what +you set up in the config file above. + +### Fortran program + +Write your Fortran tests. For functional tests, this should be an actual Fortran `program`. +For unit tests this will be either a ctest or a pfunit test. See existing tests for examples. + +For functional tests, if you output an output file, the name must match what you set up +in the config file above. + +### Python setup - functional tests + +For functional tests, you will need to add your test as a new concrete class based on +the abstract FunctionalTest class. See examples in the `functional_testing` directory. +Most of the work involves creating a plotting function for your test. + +You will then need to add this class as an import statment at the top of the +`run_functional_tests.py` script. \ No newline at end of file diff --git a/testing/cime_setup.md b/testing/cime_setup.md new file mode 100644 index 0000000000..8fd9148b59 --- /dev/null +++ b/testing/cime_setup.md @@ -0,0 +1,122 @@ +# Instructions for setting up CIME on your personal computer + +## Mac and Linux Users + +### Downloads and Installs + +1. *For Mac Users Only*: Install Apple Developer Tools, if you haven't already +2. Install homebrew ([link here](https://brew.sh/)) +3. Download ESMF ([link here](https://earthsystemmodeling.org/static/releases.html)) + +### PFunit + +Download and install pfunit using the instructions on their ([GitHub page](https://github.com/Goddard-Fortran-Ecosystem/pFUnit)) + +#### Homebrew Installs + +```bash +brew install subversion git bash-completion + +brew install kdiff3 + +brew install gcc + +brew install netcdf + +brew install nco + +brew install ncview + +brew install mpich + +brew install cmake + +brew install markdown + +brew install sloccount + +brew install pyqt --with-python3 + +brew install lapack +``` + +For compilers to find `lapack` you may need to set: + +```bash +export LDFLAGS="-L/usr/local/opt/lapack/lib" +export CPPFLAGS="-I/usr/local/opt/lapack/include" +``` + +For compilers to find `libomp` you may need to set: + +```bash +export LDFLAGS="-L/usr/local/opt/libomp/lib" +export CPPFLAGS="-I/usr/local/opt/libomp/include" +``` + +```bash +brew install highlight + +brew install git-when-merged + +brew install the_silver_searcher +``` + +*For Mac users*: `brew cask install mactex` + +### ESMF Installation + +Set an environment variable `ESMF_DIR` to where you want to install ESMF, for example: `export ESMF_DIR=/Users/afoster/esmf/esmf-8.4.0` + +Next set up some other environment variables: + +```bash +export ESMF_INSTALL_PREFIX=$ESMF_DIR/install_dir +export ESMF_NETCDF=split +export ESMF_NETCDF_INCLUDE=/usr/local/include +export ESMF_NETCDF_LIBPATH=/usr/local/lib +export ESMF_COMM=openmpi +export ESMF_COMPILER=gfortranclang +``` + +Inside the download, run: + +```bash +gmake -j4 lib +gmake install +``` + +### CIME Setup + +You'll need to set up a `.cime` directory with some specific files in it. I use Bill Sack's setup: + +```bash +cd +git clone https://github.com/billsacks/mac_cime_configuration.git .cime +git checkout -b "my_configuration" +``` + +You'll need to modify some of the files and folders. + +1. In the top-level `config_machines.xml`, change the `MACH` values to match your machine name (can obtain this via the terminal by typing `hostname`) +2. Rename the `green` directory to your computer's hostname. +3. In the `{hostname}/config_machines.xml`, update relevant fields like `MACH`, `DESC`, `OS`, and `SUPPORTED_BY` +4. In `{hostname}/config_machines.xml`, update the `ESMFMKFILE` based on the path to what you just made above. +5. Rename `gnu_green.cmake` to `gnu_{hostname}.cmake`. +6. Inside `gnu_{hostname}.cmake` update `NETCDF_C_PATH`, `NETCDF_FORTRAN_PATH`, `APPEND LDFLAGS` (both), and `PFUNIT_PATH` to your netcdf and pfunit paths (see below). + +#### Libraries + +The `NETCDF_C_PATH` should be the output of `nc-config --prefix`. +The `NETCDF_FORTRAN_PATH` should be the output of `nf-config --prefix` +Then update the `APPEND LDFLAGS` section as the output from: + +```bash +nc-config --libs +``` + +```bash +nf-config --flibs +``` + +Pfunit should be in a directiory called `installed` in the build directory. diff --git a/testing/testing_classes.py b/testing/functional_class.py similarity index 88% rename from testing/testing_classes.py rename to testing/functional_class.py index 3092df511a..6ca085ef2c 100644 --- a/testing/testing_classes.py +++ b/testing/functional_class.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod from utils import str_to_bool, str_to_list -class Heuristic(ABC): - """Class for running FATES heuristics""" +class FunctionalTest(ABC): + """Class for running FATES functional tests""" def __init__(self, name:str, test_dir:str, test_exe:str, out_file:str, use_param_file:str, other_args:str): diff --git a/testing/functional_testing/allometry/allometry_test.py b/testing/functional_testing/allometry/allometry_test.py index 7896b6678c..eb77ddc53a 100644 --- a/testing/functional_testing/allometry/allometry_test.py +++ b/testing/functional_testing/allometry/allometry_test.py @@ -5,9 +5,9 @@ import numpy as np import matplotlib.pyplot as plt from utils import round_up, get_color_palette, blank_plot -from testing_classes import Heuristic +from functional_class import FunctionalTest -class AllometryTest(Heuristic): +class AllometryTest(FunctionalTest): name = 'allometry' diff --git a/testing/functional_testing/math_utils/math_utils_test.py b/testing/functional_testing/math_utils/math_utils_test.py index fa978118a6..36308e6be0 100644 --- a/testing/functional_testing/math_utils/math_utils_test.py +++ b/testing/functional_testing/math_utils/math_utils_test.py @@ -5,9 +5,9 @@ import numpy as np import matplotlib.pyplot as plt from utils import get_color_palette -from testing_classes import Heuristic +from functional_class import FunctionalTest -class QuadraticTest(Heuristic): +class QuadraticTest(FunctionalTest): name = 'quadratic' diff --git a/testing/run_functional_tests.py b/testing/run_functional_tests.py index f0c01762ee..2e1be72195 100755 --- a/testing/run_functional_tests.py +++ b/testing/run_functional_tests.py @@ -35,7 +35,7 @@ from utils import copy_file, create_nc_from_cdl, config_to_dict, parse_test_list # add testing subclasses here -from testing_classes import Heuristic +from functional_class import FunctionalTest from functional_testing.allometry.allometry_test import AllometryTest from functional_testing.math_utils.math_utils_test import QuadraticTest @@ -362,7 +362,7 @@ def main(): """ full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) - subclasses = Heuristic.__subclasses__() + subclasses = FunctionalTest.__subclasses__() args = commandline_args() config_dict = parse_test_list(full_test_dict, args.test_list) diff --git a/testing/testing.yml b/testing/testing.yml new file mode 100644 index 0000000000..85407e9adf --- /dev/null +++ b/testing/testing.yml @@ -0,0 +1,14 @@ +name: fates_testing +channels: + - conda-forge + - defaults +dependencies: + - pip + - python=3.12 + - matplotlib + - netcdf4 + - numpy + - pandas + - scipy + - xarray + - xesmf From ddf5c74555d14a8403f7cf3cd55c632d0e429599 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Sep 2024 14:32:18 -0600 Subject: [PATCH 7/8] pythonic updates --- testing/README.testing.md | 13 +- .../allometry/allometry_test.py | 189 ++++++++++------- .../math_utils/math_utils_test.py | 44 ++-- testing/path_utils.py | 10 +- testing/run_functional_tests.py | 197 +++++++++++------- testing/run_unit_tests.py | 60 +++--- testing/utils.py | 124 +++++++---- 7 files changed, 389 insertions(+), 248 deletions(-) diff --git a/testing/README.testing.md b/testing/README.testing.md index 5275ec7816..4419b38ab4 100644 --- a/testing/README.testing.md +++ b/testing/README.testing.md @@ -38,11 +38,14 @@ First, determine if you are going to create a functional or unit test. Remember, unit tests must have a pass/fail outcome. These are best for testing edgecases, error handling, or where we know exactly what the result should be from a method. -First, add your test to either the `functional_tests.cfg` or `unit_tests.cfg` config file, depending on the test you want to create. +First, add your test to either the `functional_tests.cfg` or `unit_tests.cfg` config file, +depending on the test you want to create. ### Config file information -The `test_dir` is where cmake will place relevant libraries created from your test. The convention is to call this directory `fates_{testname}_ftest` for functional tests and `fates_{testname}_utest` for unit tests. +The `test_dir` is where cmake will place relevant libraries created from your test. +The convention is to call this directory `fates_{testname}_ftest` for functional tests +and `fates_{testname}_utest` for unit tests. The `test_exe` (only applicable for functional tests) is the executable the script will create based on your test program. The convention is to call it `FATES_{testname}_exe`. @@ -53,7 +56,8 @@ may or may not create. Set the value to `None` if your test does not create one. Set `use_param_file` to `True` if your test uses the FATES parameter file, and `False` otherwise. This is only applicable for functional tests. -Add any other arguments your test needs in the `other_args` list. This is only applicable for functional tests. +Add any other arguments your test needs in the `other_args` list. +This is only applicable for functional tests. ### Cmake setup @@ -64,7 +68,8 @@ In the file `testing/CMakeLists.txt` add your test directory to the list of test `add_subdirectory(functional_testing/my_new_test fates_new_test_ftest)` -The first argument must match your directory name, and the second must match the `test_dir` value you set up in the config file above. +The first argument must match your directory name, and the second must match the +`test_dir` value you set up in the config file above. Inside your new testing directory create a new `CMakeLists.txt` file to tell cmake how to build and compile your program. It may be easiest to copy an existing one diff --git a/testing/functional_testing/allometry/allometry_test.py b/testing/functional_testing/allometry/allometry_test.py index eb77ddc53a..bb24ab3729 100644 --- a/testing/functional_testing/allometry/allometry_test.py +++ b/testing/functional_testing/allometry/allometry_test.py @@ -1,5 +1,7 @@ +""" +Concrete class for running the allometry functional tests for FATES. +""" import os -import math import xarray as xr import pandas as pd import numpy as np @@ -7,17 +9,24 @@ from utils import round_up, get_color_palette, blank_plot from functional_class import FunctionalTest + class AllometryTest(FunctionalTest): - - name = 'allometry' - + """Quadratic test class + """ + + name = "allometry" + def __init__(self, test_dict): - super().__init__(AllometryTest.name, test_dict['test_dir'], - test_dict['test_exe'], test_dict['out_file'], - test_dict['use_param_file'], - test_dict['other_args']) + super().__init__( + AllometryTest.name, + test_dict["test_dir"], + test_dict["test_exe"], + test_dict["out_file"], + test_dict["use_param_file"], + test_dict["other_args"], + ) self.plot = True - + def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): """Plots all allometry plots @@ -32,65 +41,70 @@ def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): allometry_dat = xr.open_dataset(os.path.join(run_dir, self.out_file)) plot_dict = { - 'height': { - 'varname': 'height', - 'units': 'm', + "height": { + "varname": "height", + "units": "m", }, - 'bagw': { - 'varname': 'aboveground biomass', - 'units': 'kgC', + "bagw": { + "varname": "aboveground biomass", + "units": "kgC", }, - 'blmax': { - 'varname': 'maximum leaf biomass', - 'units': 'kgC', + "blmax": { + "varname": "maximum leaf biomass", + "units": "kgC", }, - 'crown_area': { - 'varname': 'crown area', - 'units': 'm$^2$', + "crown_area": { + "varname": "crown area", + "units": "m$^2$", }, - 'sapwood_area': { - 'varname': 'sapwood area', - 'units': 'm$^2$', + "sapwood_area": { + "varname": "sapwood area", + "units": "m$^2$", }, - 'bsap': { - 'varname': 'sapwood biomass', - 'units': 'kgC', + "bsap": { + "varname": "sapwood biomass", + "units": "kgC", }, - 'bbgw': { - 'varname': 'belowground biomass', - 'units': 'kgC', + "bbgw": { + "varname": "belowground biomass", + "units": "kgC", }, - 'fineroot_biomass': { - 'varname': 'fineroot biomass', - 'units': 'kgC', + "fineroot_biomass": { + "varname": "fineroot biomass", + "units": "kgC", }, - 'bstore': { - 'varname': 'storage biomass', - 'units': 'kgC', + "bstore": { + "varname": "storage biomass", + "units": "kgC", }, - 'bdead': { - 'varname': 'deadwood biomass', - 'units': 'kgC', + "bdead": { + "varname": "deadwood biomass", + "units": "kgC", }, - 'total_biomass_parts': { - 'varname': 'total biomass (calculated from parts)', - 'units': 'kgC', + "total_biomass_parts": { + "varname": "total biomass (calculated from parts)", + "units": "kgC", }, - 'total_biomass_tissues': { - 'varname': 'total biomass (calculated from tissues)', - 'units': 'kgC', + "total_biomass_tissues": { + "varname": "total biomass (calculated from tissues)", + "units": "kgC", }, - } for plot, attributes in plot_dict.items(): - self.plot_allometry_var(allometry_dat[plot], attributes['varname'], - attributes['units'], save_figs, plot_dir) + self.plot_allometry_var( + allometry_dat[plot], + attributes["varname"], + attributes["units"], + save_figs, + plot_dir, + ) self.plot_total_biomass(allometry_dat, save_figs, plot_dir) - + @staticmethod - def plot_allometry_var(data: xr.Dataset, varname: str, units: str, - save_fig: bool, plot_dir :str=None): + def plot_allometry_var( + data: xr.Dataset, varname: str, units: str, save_fig: bool, plot_dir: str = None + ): """Plot an allometry variable Args: @@ -101,11 +115,15 @@ def plot_allometry_var(data: xr.Dataset, varname: str, units: str, save_fig (bool): whether or not to write out plot plot_dir (str): if saving figure, where to write to """ - data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), - 'pft': np.repeat(data.pft, len(data.dbh)), - data.name: data.values.flatten()}) - - max_dbh = data_frame['dbh'].max() + data_frame = pd.DataFrame( + { + "dbh": np.tile(data.dbh, len(data.pft)), + "pft": np.repeat(data.pft, len(data.dbh)), + data.name: data.values.flatten(), + } + ) + + max_dbh = data_frame["dbh"].max() max_var = round_up(data_frame[data.name].max()) blank_plot(max_dbh, 0.0, max_var, 0.0, draw_horizontal_lines=True) @@ -114,18 +132,23 @@ def plot_allometry_var(data: xr.Dataset, varname: str, units: str, colors = get_color_palette(len(pfts)) for rank, pft in enumerate(pfts): dat = data_frame[data_frame.pft == pft] - plt.plot(dat.dbh.values, dat[data.name].values, lw=2, color=colors[rank], - label=pft) - - plt.xlabel('DBH (cm)', fontsize=11) - plt.ylabel(f'{varname} ({units})', fontsize=11) + plt.plot( + dat.dbh.values, + dat[data.name].values, + lw=2, + color=colors[rank], + label=pft, + ) + + plt.xlabel("DBH (cm)", fontsize=11) + plt.ylabel(f"{varname} ({units})", fontsize=11) plt.title(f"Simulated {varname} for input parameter file", fontsize=11) - plt.legend(loc='upper left', title='PFT') + plt.legend(loc="upper left", title="PFT") if save_fig: fig_name = os.path.join(plot_dir, f"allometry_plot_{data.name}.png") plt.savefig(fig_name) - + @staticmethod def plot_total_biomass(data: xr.Dataset, save_fig: bool, plot_dir: str): """Plot two calculations of total biomass against each other @@ -133,13 +156,19 @@ def plot_total_biomass(data: xr.Dataset, save_fig: bool, plot_dir: str): Args: data (xarray DataSet): the allometry dataset """ - data_frame = pd.DataFrame({'dbh': np.tile(data.dbh, len(data.pft)), - 'pft': np.repeat(data.pft, len(data.dbh)), - 'total_biomass_parts': data.total_biomass_parts.values.flatten(), - 'total_biomass_tissues': data.total_biomass_tissues.values.flatten()}) - - max_biomass = np.maximum(data_frame['total_biomass_parts'].max(), - data_frame['total_biomass_tissues'].max()) + data_frame = pd.DataFrame( + { + "dbh": np.tile(data.dbh, len(data.pft)), + "pft": np.repeat(data.pft, len(data.dbh)), + "total_biomass_parts": data.total_biomass_parts.values.flatten(), + "total_biomass_tissues": data.total_biomass_tissues.values.flatten(), + } + ) + + max_biomass = np.maximum( + data_frame["total_biomass_parts"].max(), + data_frame["total_biomass_tissues"].max(), + ) blank_plot(max_biomass, 0.0, max_biomass, 0.0, draw_horizontal_lines=False) @@ -147,14 +176,20 @@ def plot_total_biomass(data: xr.Dataset, save_fig: bool, plot_dir: str): colors = get_color_palette(len(pfts)) for rank, pft in enumerate(pfts): data = data_frame[data_frame.pft == pft] - plt.scatter(data.total_biomass_parts.values, data.total_biomass_parts.values, - color=colors[rank], label=pft) - - plt.xlabel('Total biomass (kgC) from parts', fontsize=11) - plt.ylabel('Total biomass (kgC) from tissues', fontsize=11) + plt.scatter( + data.total_biomass_parts.values, + data.total_biomass_parts.values, + color=colors[rank], + label=pft, + ) + + plt.xlabel("Total biomass (kgC) from parts", fontsize=11) + plt.ylabel("Total biomass (kgC) from tissues", fontsize=11) plt.title("Simulated total biomass for input parameter file", fontsize=11) - plt.legend(loc='upper left', title='PFT') + plt.legend(loc="upper left", title="PFT") if save_fig: - fig_name = os.path.join(plot_dir, "allometry_plot_total_biomass_compare.png") + fig_name = os.path.join( + plot_dir, "allometry_plot_total_biomass_compare.png" + ) plt.savefig(fig_name) diff --git a/testing/functional_testing/math_utils/math_utils_test.py b/testing/functional_testing/math_utils/math_utils_test.py index 36308e6be0..579838df18 100644 --- a/testing/functional_testing/math_utils/math_utils_test.py +++ b/testing/functional_testing/math_utils/math_utils_test.py @@ -1,23 +1,31 @@ +""" +Concrete class for running the quadtratic functional tests for FATES. +""" import os -import math import xarray as xr -import pandas as pd import numpy as np import matplotlib.pyplot as plt from utils import get_color_palette from functional_class import FunctionalTest + class QuadraticTest(FunctionalTest): - - name = 'quadratic' - + """Quadratic test class + """ + + name = "quadratic" + def __init__(self, test_dict): - super().__init__(QuadraticTest.name, test_dict['test_dir'], - test_dict['test_exe'], test_dict['out_file'], - test_dict['use_param_file'], - test_dict['other_args']) + super().__init__( + QuadraticTest.name, + test_dict["test_dir"], + test_dict["test_exe"], + test_dict["out_file"], + test_dict["use_param_file"], + test_dict["other_args"], + ) self.plot = True - + def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): """Reads in and plots quadratic formula test output @@ -32,13 +40,17 @@ def plot_output(self, run_dir: str, save_figs: bool, plot_dir: str): quadratic_dat = xr.open_dataset(os.path.join(run_dir, self.out_file)) # plot output - self.plot_quad_and_roots(quadratic_dat.a.values, quadratic_dat.b.values, - quadratic_dat.c.values, quadratic_dat.root1.values, - quadratic_dat.root2.values) + self.plot_quad_and_roots( + quadratic_dat.a.values, + quadratic_dat.b.values, + quadratic_dat.c.values, + quadratic_dat.root1.values, + quadratic_dat.root2.values, + ) if save_figs: fig_name = os.path.join(plot_dir, "quadratic_test.png") plt.savefig(fig_name) - + @staticmethod def plot_quad_and_roots(a_coeff, b_coeff, c_coeff, root1, root2): """Plots a set of quadratic formulas (ax**2 + bx + c) and their two roots @@ -57,7 +69,7 @@ def plot_quad_and_roots(a_coeff, b_coeff, c_coeff, root1, root2): colors = get_color_palette(num_equations) for i in range(num_equations): - y_vals = a_coeff[i]*x_vals**2 + b_coeff[i]*x_vals + c_coeff[i] + y_vals = a_coeff[i] * x_vals**2 + b_coeff[i] * x_vals + c_coeff[i] plt.plot(x_vals, y_vals, lw=2, color=colors[i]) plt.scatter(root1[i], root2[i], color=colors[i], s=50) - plt.axhline(y=0.0, color='k', linestyle='dotted') \ No newline at end of file + plt.axhline(y=0.0, color="k", linestyle="dotted") diff --git a/testing/path_utils.py b/testing/path_utils.py index 3a5e533606..0343a69f1b 100644 --- a/testing/path_utils.py +++ b/testing/path_utils.py @@ -10,6 +10,7 @@ os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) ) + def add_cime_lib_to_path() -> str: """Adds the CIME python library to the python path, to allow importing modules from that library @@ -19,12 +20,13 @@ def add_cime_lib_to_path() -> str: """ cime_path = path_to_cime() prepend_to_python_path(cime_path) - + cime_lib_path = os.path.join(cime_path, "CIME", "Tools") prepend_to_python_path(cime_lib_path) - + return cime_path + def path_to_cime() -> str: """Returns the path to cime, if it can be found @@ -39,6 +41,7 @@ def path_to_cime() -> str: return cime_path raise RuntimeError("Cannot find cime.") + def path_to_fates_root(): """Returns Returns the path to the root directory of FATES @@ -47,7 +50,8 @@ def path_to_fates_root(): """ return _FATES_ROOT -def prepend_to_python_path(path:str): + +def prepend_to_python_path(path: str): """Adds the given path to python's sys.path if not already there Path is added near the beginning, so that it takes precedence over existing entries in path. diff --git a/testing/run_functional_tests.py b/testing/run_functional_tests.py index 2e1be72195..5f80a3f095 100755 --- a/testing/run_functional_tests.py +++ b/testing/run_functional_tests.py @@ -19,7 +19,7 @@ You must also have a .cime folder in your home directory which specifies machine configurations for CIME. -This script builds and runs FATES functional tests, and plots any relevant output from +This script builds and runs FATES functional tests, and plots any relevant output from those tests. You can supply your own parameter file (either a .cdl or a .nc file), or if you do not @@ -36,12 +36,12 @@ # add testing subclasses here from functional_class import FunctionalTest -from functional_testing.allometry.allometry_test import AllometryTest -from functional_testing.math_utils.math_utils_test import QuadraticTest +from functional_testing.allometry.allometry_test import AllometryTest # pylint: disable=unused-import +from functional_testing.math_utils.math_utils_test import QuadraticTest # pylint: disable=unused-import add_cime_lib_to_path() -from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order +from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order # constants for this script _DEFAULT_CONFIG_FILE = "functional_tests.cfg" @@ -49,6 +49,7 @@ _CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") _TEST_SUB_DIR = "testing" + def commandline_args(): """Parse and return command-line arguments""" @@ -109,44 +110,42 @@ def commandline_args(): ) parser.add_argument( - "--skip-build", - action="store_true", - help="Skip building and compiling the test code.\n" - "Only do this if you already have run build.\n" - "Script will check to make sure executables are present.\n", + "--skip-build", + action="store_true", + help="Skip building and compiling the test code.\n" + "Only do this if you already have run build.\n" + "Script will check to make sure executables are present.\n", ) parser.add_argument( - "--skip-run-executables", - action="store_true", - help="Skip running test code executables.\n" - "Only do this if you already have run the code previously.\n" - "Script will check to make sure required output files are present.\n", + "--skip-run-executables", + action="store_true", + help="Skip running test code executables.\n" + "Only do this if you already have run the code previously.\n" + "Script will check to make sure required output files are present.\n", ) parser.add_argument( - "--save-figs", - action="store_true", - help="Write out generated figures to files.\n" - "Will be placed in run_dir/plots.\n" - "Should probably do this on remote machines.\n", + "--save-figs", + action="store_true", + help="Write out generated figures to files.\n" + "Will be placed in run_dir/plots.\n" + "Should probably do this on remote machines.\n", ) - + parser.add_argument( - "--verbose-make", - action="store_true", - help="Run make with verbose output." + "--verbose-make", action="store_true", help="Run make with verbose output." ) parser.add_argument( - "-t", - "--test-list", - action="store", - dest="test_list", - type=str, - default="all", - help="Test(s) to run. Comma-separated list of test names, or 'all'\n" - "for all tests. If not supplied, will run all tests." + "-t", + "--test-list", + action="store", + dest="test_list", + type=str, + default="all", + help="Test(s) to run. Comma-separated list of test names, or 'all'\n" + "for all tests. If not supplied, will run all tests.", ) args = parser.parse_args() @@ -155,6 +154,7 @@ def commandline_args(): return args + def check_arg_validity(args): """Checks validity of input script arguments @@ -163,7 +163,7 @@ def check_arg_validity(args): """ # check to make sure parameter file exists and is one of the correct forms check_param_file(args.parameter_file) - + # make sure relevant output files exist: if args.skip_run_executables: # if you skip the run we assume you want to skip the build @@ -174,10 +174,14 @@ def check_arg_validity(args): # make sure build directory exists if args.skip_build: if args.verbose_make: - raise argparse.ArgumentError(None, "Can't run verbose make and skip build.\n" - "Re-run script without --skip-build") + raise argparse.ArgumentError( + None, + "Can't run verbose make and skip build.\n" + "Re-run script without --skip-build", + ) check_build_dir(args.build_dir, args.test_dict) + def check_param_file(param_file): """Checks to see if param_file exists and is of the correct form (.nc or .cdl) @@ -189,11 +193,14 @@ def check_param_file(param_file): argparse.ArgumentError: Can't find parameter file """ file_suffix = os.path.basename(param_file).split(".")[-1] - if not file_suffix in ['cdl', 'nc']: - raise argparse.ArgumentError(None, "Must supply parameter file with .cdl or .nc ending.") + if not file_suffix in ["cdl", "nc"]: + raise argparse.ArgumentError( + None, "Must supply parameter file with .cdl or .nc ending." + ) if not os.path.isfile(param_file): raise argparse.ArgumentError(None, f"Cannot find file {param_file}.") - + + def check_build_dir(build_dir, test_dict): """Checks to see if all required build directories and executables are present @@ -205,10 +212,14 @@ def check_build_dir(build_dir, test_dict): argparse.ArgumentError: Can't find a required build directory or executable """ for attributes in test_dict.values(): - if not build_exists(build_dir, attributes['test_dir'], attributes['test_exe']): - raise argparse.ArgumentError(None, "Build directory or executable does not exist.\n" - "Re-run script without --skip-build.") - + if not build_exists(build_dir, attributes["test_dir"], attributes["test_exe"]): + raise argparse.ArgumentError( + None, + "Build directory or executable does not exist.\n" + "Re-run script without --skip-build.", + ) + + def check_out_files(run_dir, test_dict): """Checks to see that required output files are present in the run directory @@ -219,14 +230,31 @@ def check_out_files(run_dir, test_dict): Raises: argparse.ArgumentError: Can't find a required output file """ - for test, attributes in dict(filter(lambda pair: pair[1]['out_file'] is not None, - test_dict.items())).items(): - if not os.path.isfile(os.path.join(os.path.abspath(run_dir), attributes['out_file'])): - raise argparse.ArgumentError(None, f"Required file for {test} test does not exist.\n" - "Re-run script without --skip-run.") - -def run_functional_tests(clean, verbose_make, build, run_executables, build_dir, run_dir, - make_j, param_file, save_figs, test_dict): + for test, attributes in dict( + filter(lambda pair: pair[1]["out_file"] is not None, test_dict.items()) + ).items(): + if not os.path.isfile( + os.path.join(os.path.abspath(run_dir), attributes["out_file"]) + ): + raise argparse.ArgumentError( + None, + f"Required file for {test} test does not exist.\n" + "Re-run script without --skip-run.", + ) + + +def run_functional_tests( + clean, + verbose_make, + build, + run_executables, + build_dir, + run_dir, + make_j, + param_file, + save_figs, + test_dict, +): """Builds and runs the fates functional tests Args: @@ -261,8 +289,9 @@ def run_functional_tests(clean, verbose_make, build, run_executables, build_dir, # compile code if build: - build_tests(build_dir, _CMAKE_BASE_DIR, make_j, clean=clean, - verbose=verbose_make) + build_tests( + build_dir, _CMAKE_BASE_DIR, make_j, clean=clean, verbose=verbose_make + ) # run executables for each test in test list if run_executables: @@ -273,17 +302,21 @@ def run_functional_tests(clean, verbose_make, build, run_executables, build_dir, if test.use_param_file: args.insert(0, param_file) # run - run_fortran_exectuables(build_dir_path, test.test_dir, - test.test_exe, run_dir_path, args) - + run_fortran_exectuables( + build_dir_path, test.test_dir, test.test_exe, run_dir_path, args + ) + # plot output for relevant tests - for name, test in dict(filter(lambda pair: pair[1].plot, - test_dict.items())).items(): - test.plot_output(run_dir_path, save_figs, - os.path.join(run_dir_path, 'plots', name)) + for name, test in dict( + filter(lambda pair: pair[1].plot, test_dict.items()) + ).items(): + test.plot_output( + run_dir_path, save_figs, os.path.join(run_dir_path, "plots", name) + ) # show plots plt.show() - + + def make_plotdirs(run_dir, test_dict): """Create plotting directories if they don't already exist @@ -292,16 +325,17 @@ def make_plotdirs(run_dir, test_dict): test_dict (dict): dictionary of test to run """ # make main plot directory - plot_dir = os.path.join(run_dir, 'plots') + plot_dir = os.path.join(run_dir, "plots") if not os.path.isdir(plot_dir): os.mkdir(plot_dir) - + # make sub-plot directories for test in dict(filter(lambda pair: pair[1].plot, test_dict.items())): sub_dir = os.path.join(plot_dir, test) if not os.path.isdir(sub_dir): os.mkdir(sub_dir) - + + def create_param_file(param_file, run_dir): """Creates and/or move the default or input parameter file to the run directory Creates a netcdf file from a cdl file if a cdl file is supplied @@ -323,7 +357,7 @@ def create_param_file(param_file, run_dir): else: print(f"Using parameter file {param_file}.") file_suffix = os.path.basename(param_file).split(".")[-1] - if file_suffix == 'cdl': + if file_suffix == "cdl": param_file_update = create_nc_from_cdl(param_file, run_dir) elif file_suffix == "nc": param_file_update = copy_file(param_file, run_dir) @@ -332,6 +366,7 @@ def create_param_file(param_file, run_dir): return param_file_update + def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): """Run the generated Fortran executables @@ -355,30 +390,42 @@ def run_fortran_exectuables(build_dir, test_dir, test_exe, run_dir, args): os.chdir(run_dir) out = run_cmd_no_fail(" ".join(run_command), combine_output=True) print(out) - + + def main(): """Main script - Reads in command-line arguments and then runs the tests. + Reads in command-line arguments and then runs the tests. """ - + full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) subclasses = FunctionalTest.__subclasses__() - + args = commandline_args() config_dict = parse_test_list(full_test_dict, args.test_list) - - # for now just turn into a dictionary? + test_dict = {} for name in config_dict.keys(): - test_class = list(filter(lambda subclass: subclass.name == name, subclasses))[0](config_dict[name]) + test_class = list(filter(lambda subclass: subclass.name == name, subclasses))[ + 0 + ](config_dict[name]) test_dict[name] = test_class build = not args.skip_build run = not args.skip_run_executables - - run_functional_tests(args.clean, args.verbose_make, build, run, args.build_dir, - args.run_dir, args.make_j, args.parameter_file, args.save_figs, - test_dict) - + + run_functional_tests( + args.clean, + args.verbose_make, + build, + run, + args.build_dir, + args.run_dir, + args.make_j, + args.parameter_file, + args.save_figs, + test_dict, + ) + + if __name__ == "__main__": main() diff --git a/testing/run_unit_tests.py b/testing/run_unit_tests.py index 2656bda0e5..f9bd344b39 100755 --- a/testing/run_unit_tests.py +++ b/testing/run_unit_tests.py @@ -26,13 +26,14 @@ add_cime_lib_to_path() -from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order +from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order # constants for this script _CMAKE_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") _DEFAULT_CONFIG_FILE = "unit_tests.cfg" _TEST_SUB_DIR = "testing" + def commandline_args(): """Parse and return command-line arguments""" @@ -73,26 +74,25 @@ def commandline_args(): ) parser.add_argument( - "--verbose-make", - action="store_true", - help="Run make with verbose output." + "--verbose-make", action="store_true", help="Run make with verbose output." ) - + parser.add_argument( - "-t", - "--test-list", - action="store", - dest="test_list", - type=str, - default="all", - help="Test(s) to run. Comma-separated list of test names, or 'all'\n" - "for all tests. If not supplied, will run all tests." + "-t", + "--test-list", + action="store", + dest="test_list", + type=str, + default="all", + help="Test(s) to run. Comma-separated list of test names, or 'all'\n" + "for all tests. If not supplied, will run all tests.", ) args = parser.parse_args() return args + def run_unit_tests(clean, verbose_make, build_dir, make_j, test_dict): """Builds and runs the fates unit tests @@ -108,30 +108,36 @@ def run_unit_tests(clean, verbose_make, build_dir, make_j, test_dict): build_dir_path = os.path.abspath(build_dir) # compile code - build_tests(build_dir_path, _CMAKE_BASE_DIR, make_j, clean=clean, - verbose=verbose_make) - + build_tests( + build_dir_path, _CMAKE_BASE_DIR, make_j, clean=clean, verbose=verbose_make + ) + # run unit tests - print(f"Running unit tests...") + print("Running unit tests...") for _, attributes in test_dict.items(): - - test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes['test_dir']) + + test_dir = os.path.join(build_dir_path, _TEST_SUB_DIR, attributes["test_dir"]) ctest_command = ["ctest", "--output-on-failure"] - output = run_cmd_no_fail(" ".join(ctest_command), from_dir=test_dir, - combine_output=True) + output = run_cmd_no_fail( + " ".join(ctest_command), from_dir=test_dir, combine_output=True + ) print(output) - + + def main(): """Main script - Reads in command-line arguments and then runs the tests. + Reads in command-line arguments and then runs the tests. """ - + full_test_dict = config_to_dict(_DEFAULT_CONFIG_FILE) args = commandline_args() test_dict = parse_test_list(full_test_dict, args.test_list) - - run_unit_tests(args.clean, args.verbose_make, args.build_dir, args.make_j, test_dict) - + + run_unit_tests( + args.clean, args.verbose_make, args.build_dir, args.make_j, test_dict + ) + + if __name__ == "__main__": main() diff --git a/testing/utils.py b/testing/utils.py index d6201bdceb..d1fafd5838 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -10,9 +10,10 @@ add_cime_lib_to_path() -from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order +from CIME.utils import run_cmd_no_fail # pylint: disable=wrong-import-position,import-error,wrong-import-order -def round_up(num:float, decimals:int=0) -> float: + +def round_up(num: float, decimals: int = 0) -> float: """Rounds a number up Args: @@ -23,9 +24,10 @@ def round_up(num:float, decimals:int=0) -> float: float: input number rounded up """ multiplier = 10**decimals - return math.ceil(num * multiplier)/multiplier + return math.ceil(num * multiplier) / multiplier + -def truncate(num:float, decimals:int=0) -> float: +def truncate(num: float, decimals: int = 0) -> float: """Rounds a number down Args: @@ -36,9 +38,10 @@ def truncate(num:float, decimals:int=0) -> float: float: number rounded down """ multiplier = 10**decimals - return int(num * multiplier)/multiplier + return int(num * multiplier) / multiplier + -def create_nc_from_cdl(cdl_path:str, run_dir:str) -> str: +def create_nc_from_cdl(cdl_path: str, run_dir: str) -> str: """Creates a netcdf file from a cdl file and return path to new file. Args: @@ -48,17 +51,14 @@ def create_nc_from_cdl(cdl_path:str, run_dir:str) -> str: file_basename = os.path.basename(cdl_path).split(".")[-2] file_nc_name = f"{file_basename}.nc" - file_gen_command = [ - "ncgen -o", - os.path.join(run_dir, file_nc_name), - cdl_path - ] + file_gen_command = ["ncgen -o", os.path.join(run_dir, file_nc_name), cdl_path] out = run_cmd_no_fail(" ".join(file_gen_command), combine_output=True) print(out) return file_nc_name -def copy_file(file_path:str, directory) -> str: + +def copy_file(file_path: str, directory) -> str: """Copies a file file to a desired directory and returns path to file. Args: @@ -67,17 +67,14 @@ def copy_file(file_path:str, directory) -> str: """ file_basename = os.path.basename(file_path) - file_copy_command = [ - "cp", - os.path.abspath(file_path), - os.path.abspath(directory) - ] + file_copy_command = ["cp", os.path.abspath(file_path), os.path.abspath(directory)] out = run_cmd_no_fail(" ".join(file_copy_command), combine_output=True) print(out) return file_basename -def get_color_palette(number:int) -> list: + +def get_color_palette(number: int) -> list: """_summary_ Args: @@ -89,22 +86,42 @@ def get_color_palette(number:int) -> list: Returns: list[tuple]: list of colors to use in plotting """ - + # hard-coded list of colors, can add more here if necessary - all_colors = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120), - (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150), - (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148), - (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199), - (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)] - + all_colors = [ + (31, 119, 180), + (174, 199, 232), + (255, 127, 14), + (255, 187, 120), + (44, 160, 44), + (152, 223, 138), + (214, 39, 40), + (255, 152, 150), + (148, 103, 189), + (197, 176, 213), + (140, 86, 75), + (196, 156, 148), + (227, 119, 194), + (247, 182, 210), + (127, 127, 127), + (199, 199, 199), + (188, 189, 34), + (219, 219, 141), + (23, 190, 207), + (158, 218, 229), + ] + if number > len(all_colors): raise ValueError(f"get_color_palette: number must be <= {len(all_colors)}") - colors = [(red/255.0, green/255.0, blue/255.0) for red, green, blue in all_colors] + colors = [ + (red / 255.0, green / 255.0, blue / 255.0) for red, green, blue in all_colors + ] return colors[:number] -def config_to_dict(config_file:str) -> dict: + +def config_to_dict(config_file: str) -> dict: """Convert a config file to a python dictionary Args: @@ -121,9 +138,10 @@ def config_to_dict(config_file:str) -> dict: dictionary[section] = {} for option in config.options(section): dictionary[section][option] = config.get(section, option) - + return dictionary + def parse_test_list(full_test_dict, test_string): """Parses the input test list and checks for errors @@ -139,20 +157,23 @@ def parse_test_list(full_test_dict, test_string): valid_test_names = full_test_dict.keys() if test_string != "all": - test_list = test_string.split(',') + test_list = test_string.split(",") for test in test_list: if test not in valid_test_names: - raise argparse.ArgumentTypeError("Invalid test supplied, \n" - "must supply one of:\n" - f"{', '.join(valid_test_names)}\n" - "or do not supply a test name to run all tests.") + raise argparse.ArgumentTypeError( + "Invalid test supplied, \n" + "must supply one of:\n" + f"{', '.join(valid_test_names)}\n" + "or do not supply a test name to run all tests." + ) test_dict = {key: full_test_dict[key] for key in test_list} else: test_dict = full_test_dict return test_dict -def str_to_bool(val:str) -> bool: + +def str_to_bool(val: str) -> bool: """Convert a string representation of truth to True or False. Args: @@ -164,13 +185,14 @@ def str_to_bool(val:str) -> bool: Returns: bool: True or False """ - if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): + if val.lower() in ("y", "yes", "t", "true", "on", "1"): return True - if val.lower() in ('n', 'no', 'f', 'false', 'off', '0'): + if val.lower() in ("n", "no", "f", "false", "off", "0"): return False raise ValueError(f"invalid truth value {val}") -def str_to_list(val:str) -> list: + +def str_to_list(val: str) -> list: """converts string representation of list to actual list Args: @@ -179,14 +201,20 @@ def str_to_list(val:str) -> list: Returns: list: actual list """ - if val in ('', '[]'): + if val in ("", "[]"): # empty list return [] - res = val.strip('][').split(',') + res = val.strip("][").split(",") return [n.strip() for n in res] -def blank_plot(x_max: float, x_min: float, y_max: float, y_min: float, - draw_horizontal_lines :bool=False): + +def blank_plot( + x_max: float, + x_min: float, + y_max: float, + y_min: float, + draw_horizontal_lines: bool = False, +): """Generate a blank plot with set attributes Args: @@ -215,11 +243,15 @@ def blank_plot(x_max: float, x_min: float, y_max: float, y_min: float, plt.xticks(fontsize=10) if draw_horizontal_lines: - inc = (int(y_max) - y_min)/20 + inc = (int(y_max) - y_min) / 20 for i in range(0, 20): - plt.plot(range(math.floor(x_min), math.ceil(x_max)), - [0.0 + i*inc] * len(range(math.floor(x_min), math.ceil(x_max))), - "--", lw=0.5, color="black", alpha=0.3) + plt.plot( + range(math.floor(x_min), math.ceil(x_max)), + [0.0 + i * inc] * len(range(math.floor(x_min), math.ceil(x_max))), + "--", + lw=0.5, + color="black", + alpha=0.3, + ) plt.tick_params(bottom=False, top=False, left=False, right=False) - From d10f218f5c27807099fdbf7ea522ba2e5209cfff Mon Sep 17 00:00:00 2001 From: adrifoster Date: Tue, 10 Sep 2024 16:24:52 -0600 Subject: [PATCH 8/8] update readme --- testing/README.testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/README.testing.md b/testing/README.testing.md index 4419b38ab4..812891d2a0 100644 --- a/testing/README.testing.md +++ b/testing/README.testing.md @@ -94,4 +94,4 @@ the abstract FunctionalTest class. See examples in the `functional_testing` dire Most of the work involves creating a plotting function for your test. You will then need to add this class as an import statment at the top of the -`run_functional_tests.py` script. \ No newline at end of file +`run_functional_tests.py` script.