diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ed6ad8a06f5..6329c5295768 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,13 +2,15 @@ cmake_minimum_required(VERSION 2.8) include(ExternalProject) set(CIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") +list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) +include(CIME_initial_setup) + project(cime_tests Fortran C) # We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't # need to duplicate this cmake code list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake") -list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) include(CIME_utils) find_package(NetCDF COMPONENTS C Fortran) include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) @@ -21,14 +23,21 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) # ------------------------------------------------------------------------ set(MCT_ROOT "${CIME_ROOT}/src/externals/mct") +if (USE_MPI_SERIAL) + set(ENABLE_MPI_SERIAL "--enable-mpiserial") +else() + set(ENABLE_MPI_SERIAL "") +endif() + ExternalProject_add(mct_project PREFIX ${CMAKE_CURRENT_BINARY_DIR} SOURCE_DIR ${MCT_ROOT} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/mct - CONFIGURE_COMMAND ${MCT_ROOT}/configure --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g" - BUILD_COMMAND $(MAKE) + CONFIGURE_COMMAND ${MCT_ROOT}/configure ${ENABLE_MPI_SERIAL} --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g" + BUILD_COMMAND $(MAKE) SRCDIR=${MCT_ROOT} # Leave things in rather than "installing", because we have - # no need to move things around inside of the CMake binary directory. + # no need to move things around inside of the CMake binary directory. Also, + # mpi-serial doesn't install properly in the out-of-source build INSTALL_COMMAND : ) # This copy_makefiles step is needed because mct currently doesn't support an @@ -43,12 +52,27 @@ ExternalProject_add_step(mct_project copy_makefiles COMMAND mkdir -p mpeu COMMAND cp -p /mpeu/Makefile mpeu/ ) +if (USE_MPI_SERIAL) + ExternalProject_add_step(mct_project copy_mpi_serial_files + DEPENDEES configure + DEPENDERS build + WORKING_DIRECTORY + COMMAND mkdir -p mpi-serial + COMMAND cp -p /mpi-serial/Makefile mpi-serial/ + COMMAND cp /mpi-serial/mpif.h mpi-serial/ + COMMAND cp /mpi-serial/mpi.h mpi-serial/ + ) +endif() # Tell cmake to look for libraries & mod files here, because this is where we built libraries include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) +if (USE_MPI_SERIAL) + include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) + link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) +endif() # ------------------------------------------------------------------------ # Done MCT build diff --git a/ChangeLog b/ChangeLog index e26826d94378..aa0351fc0c9d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,659 @@ ====================================================================== +Originator: Chris Fischer +Date: 5-2-207 +Tag: cime5.3.0-alpha.10 +Answer Changes: [None, Round Off, Climate Changing] +Tests: scripts_regression_tests.py with PIO_VERSION set to 2 in driver config_components.xml + scripts_regression test + ERP_Ln9.f09_f09.F1850_DONOTUSE.yellowstone_intel.cam-outfrq9s_clm5 + summarize_cprnc_diffs by hand + code-checker +Dependencies: + +Brief Summary: + - Update to version 2.2.1 of the pio external. + - Update the top-level READMEs. + - New optional ndep stream from atm. + - Creation of new aquaplanet capability in DOCN. + - Fix typo in parse_args. + - All command line arguments should conform. Allowed for one exception, the -value option to xmlquery. + - ERP fix again. + - Pick up multi-instance cprnc.out files in summarize_cprnc_diffs. + - Fix an issue with pio1 determining if netcdf4 is available and removes a debug print statement. + - Improve error message when config_files.xml is not found. + - Fix pylint error. + - Put explicit dot in paths in some instructions. + - Remove scripts adjustment of pio settings. + + +User interface changes: + - Now accepting only conforming command line arguments. + +PR summary: git log --oneline --first-parent [previous_tag]..master +0dcd36c Merge pull request #1447 from jedwards4b/pio2_external_update +357ad45 Merge pull request #1432 from ESMCI/rljacob/README-cleanup +c76d946 Merge pull request #1448 from ESMCI/mvertens/ndep +e5f144d Merge pull request #1451 from ESMCI/mvertens/aquap +d4dfbf1 Merge pull request #1452 from jedwards4b/acme_run_template_fix +505ce67 Merge pull request #1393 from jedwards4b/test_cli +c1e5a38 Merge pull request #1445 from jedwards4b/erp_fix_again +f6c05d1 Merge pull request #1442 from billsacks/summarize_cprnc_multiinst +71cc74e Merge pull request #1425 from jedwards4b/pio1_fixes +aa9018a Merge pull request #1440 from jedwards4b/better_msg_for_config_files_not_found +1089e53 Merge pull request #1446 from ESMCI/jgfouca/fix_pylint_err_nml +3a498d2 Merge pull request #1371 from billsacks/source_file_from_path +ae2a5ca Add Sandia Corporation to CIME LICENSE.TXT +7692745 Merge pull request #1441 from jedwards4b/pio_default_settings + + +Modified files: git diff --name-status [previous_tag] + +M LICENSE.TXT +D README +M README.md +M config/acme/machines/template.case.run +M config/acme/machines/template.case.test +M config/acme/machines/template.lt_archive +M config/acme/machines/template.st_archive +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/template.case.run +M config/cesm/machines/template.case.test +M config/cesm/machines/template.lt_archive +M config/cesm/machines/template.st_archive +M scripts/Tools/acme_check_env +M scripts/Tools/bless_test_results +M scripts/Tools/case.build +M scripts/Tools/case.cmpgen_namelists +M scripts/Tools/case.setup +M scripts/Tools/case.submit +M scripts/Tools/case_diff +M scripts/Tools/check_case +M scripts/Tools/check_input_data +M scripts/Tools/check_lockedfiles +M scripts/Tools/cime_bisect +M scripts/Tools/code_checker +M scripts/Tools/compare_namelists +M scripts/Tools/compare_test_results +M scripts/Tools/component_compare_baseline +M scripts/Tools/component_compare_copy +M scripts/Tools/component_compare_test +M scripts/Tools/component_generate_baseline +M scripts/Tools/getTiming +M scripts/Tools/jenkins_generic_job +M scripts/Tools/list_acme_tests +M scripts/Tools/normalize_cases +M scripts/Tools/pelayout +M scripts/Tools/preview_namelists +M scripts/Tools/save_provenance +M scripts/Tools/simple_compare +M scripts/Tools/testreporter.py +M scripts/Tools/update_acme_tests +M scripts/Tools/wait_for_tests +M scripts/Tools/xmlchange +M scripts/Tools/xmlconvertors/config_pes_converter.py +M scripts/Tools/xmlconvertors/grid_xml_converter.py +M scripts/Tools/xmlquery +M scripts/create_clone +M scripts/create_newcase +M scripts/create_test +M scripts/fortran_unit_testing/run_tests.py +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/SystemTests/seq.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/files.py +M scripts/lib/CIME/buildlib.py +M scripts/lib/CIME/buildnml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/utils.py +M scripts/manage_case +M scripts/manage_pes +M scripts/query_testlists +M scripts/tests/scripts_regression_tests.py +M src/build_scripts/buildlib.gptl +M src/build_scripts/buildlib.pio +M src/components/data_comps/docn/cime_config/buildnml +M src/components/data_comps/docn/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +M src/components/data_comps/docn/docn_comp_mod.F90 +M src/drivers/mct/cime_config/namelist_definition_drv_flds.xml +M src/drivers/mct/shr/seq_flds_mod.F90 +A src/drivers/mct/shr/shr_ndep_mod.F90 +M src/externals/pio1/pio/CMakeLists.txt +M src/externals/pio1/pio/pionfput_mod.F90.in +M src/externals/pio2/CMakeLists.txt +M src/externals/pio2/CTestScript.cmake +M src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +M src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh +M src/externals/pio2/ctest/runcdash-cgd-nag.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +M src/externals/pio2/ctest/runcdash-nwscla-intel.sh +M src/externals/pio2/ctest/runctest-cgd.sh +M src/externals/pio2/ctest/runctest-nwscla.sh +M src/externals/pio2/doc/source/contributing_code.txt +M src/externals/pio2/examples/c/CMakeLists.txt +A src/externals/pio2/examples/c/darray_async.c +A src/externals/pio2/examples/c/darray_no_async.c +M src/externals/pio2/examples/c/example1.c +M src/externals/pio2/src/clib/CMakeLists.txt +M src/externals/pio2/src/clib/bget.c +M src/externals/pio2/src/clib/config.h.in +M src/externals/pio2/src/clib/pio.h +M src/externals/pio2/src/clib/pio_darray.c +M src/externals/pio2/src/clib/pio_darray_int.c +M src/externals/pio2/src/clib/pio_file.c +M src/externals/pio2/src/clib/pio_get_nc.c +M src/externals/pio2/src/clib/pio_getput_int.c +M src/externals/pio2/src/clib/pio_internal.h +M src/externals/pio2/src/clib/pio_lists.c +M src/externals/pio2/src/clib/pio_msg.c +M src/externals/pio2/src/clib/pio_nc.c +M src/externals/pio2/src/clib/pio_nc4.c +M src/externals/pio2/src/clib/pio_put_nc.c +M src/externals/pio2/src/clib/pio_rearrange.c +M src/externals/pio2/src/clib/pio_spmd.c +M src/externals/pio2/src/clib/pio_varm.c +M src/externals/pio2/src/clib/pioc.c +M src/externals/pio2/src/clib/pioc_sc.c +M src/externals/pio2/src/clib/pioc_support.c +M src/externals/pio2/tests/cunit/CMakeLists.txt +M src/externals/pio2/tests/cunit/pio_tests.h +D src/externals/pio2/tests/cunit/test_async_2comp.c +M src/externals/pio2/tests/cunit/test_async_3proc.c +M src/externals/pio2/tests/cunit/test_async_4proc.c +M src/externals/pio2/tests/cunit/test_async_simple.c +M src/externals/pio2/tests/cunit/test_darray_1d.c +M src/externals/pio2/tests/cunit/test_darray_3d.c +A src/externals/pio2/tests/cunit/test_darray_multi.c +M src/externals/pio2/tests/cunit/test_darray_multivar.c +A src/externals/pio2/tests/cunit/test_darray_multivar2.c +A src/externals/pio2/tests/cunit/test_decomp_uneven.c +M src/externals/pio2/tests/cunit/test_decomps.c +M src/externals/pio2/tests/cunit/test_intercomm2.c +M src/externals/pio2/tests/cunit/test_iosystem2_simple.c +M src/externals/pio2/tests/cunit/test_pioc.c +M src/externals/pio2/tests/cunit/test_pioc_fill.c +A src/externals/pio2/tests/cunit/test_rearr.c +M src/externals/pio2/tests/cunit/test_shared.c +M src/externals/pio2/tests/cunit/test_spmd.c +M src/externals/pio2/tests/general/ncdf_get_put.F90.in +M src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +M src/externals/pio2/tests/general/util/pio_tutil.F90 +M src/share/util/shr_pio_mod.F90 +M src/share/util/shr_strdata_mod.F90 +M tools/configure +M tools/cprnc/README +M tools/cprnc/summarize_cprnc_diffs +M tools/mapping/check_maps/.gitignore +M tools/mapping/check_maps/README +M tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 +M tools/mapping/check_maps/src/Makefile +M tools/mapping/gen_domain_files/INSTALL +M tools/mapping/gen_domain_files/src/Makefile +M tools/mapping/gen_mapping_files/README +A tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore +M tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README +M tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh +M tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL +M tools/mapping/map_field/INSTALL +M tools/mapping/map_field/src/Makefile + +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-30-2017 +Tag: cime5.3.0-alpha.09 +Answer Changes: None +Tests: ERP_Ln9.f09_f09.F1850_DONOTUSE.yellowstone_intel.cam-outfrq9s_clm5 + scripts_regression_tests.py --fast, hand test xmlchange and xmlquery +Dependencies: + +Brief Summary: + - Fix erp test build. + - Add Support for '+=' in Namelists. + - Add --component Option to preview_namelist. + - Add --file option to xmlquery and enable option in xmlchange. + +User interface changes: + - Added support for '+=' to namelist parser. + - Added --component option to preview_namelist. + - New --file option in xmlquery. + +PR summary: git log --oneline --first-parent [previous_tag]..master +6d8b24b Merge pull request #1444 from jedwards4b/erp_test_fix +84b7bde Merge pull request #1389 from ESMCI/erichlf/namelist/plusEqual +45af658 Merge pull request #1436 from ESMCI/erichlf/preview_namelists/component +c4d4437 Merge pull request #1419 from jedwards4b/xmlcq_fileoption + + +Modified files: git diff --name-status [previous_tag] + +M scripts/Tools/preview_namelists +M scripts/Tools/xmlchange +M scripts/Tools/xmlquery +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/env_test.py +M scripts/lib/CIME/XML/generic_xml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/preview_namelists.py + +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-28-2017 +Tag: cime5.3.0-alpha.08 +Answer Changes: [None, Round Off, Climate Changing] +Tests: scripts_regression_tests, SMS_D_Ld3.f45_g37_rx1.A.hobart_nag, + ERS_Ln9.T62_g37.DTEST.yellowstone_intel, CESM prealpha on hobart_nag +Dependencies: cime_config0.0.1.alpha.07, cism2_1_30, clm4_5_15_r235 + +Brief Summary: + - Use ESMF7 on yellowstone and cheyenne. + - Pass annual-average fields to GLC. + - Give users more info and control over key run commands. + - Moves unit testing infrastructure from tools/unit_testing to + scripts/fortran_unit_testing. + - There were tabs in create_newcase. None prevented the script from + running, but should be removed for consistency. + - nag compliler needs an additional library flag when built with debug. + - Update case members in case_setup. + - Handle the case when executable is None. + - Fix big mistake in scripts_regression_tests. + - Minor changes to xmlchange. + - Fix minor bug in test_scheduler. + - Rewrite testreporter into python. + - Print time built per model. + - Add input-dir to create_newcase. + - Get unit test build and run working with serial or parallel pFUnit. + - Add Support of true and false in Namelist. + - Remove the cprnc.out files prior to running tests. + - Fix bug where LockedFiles weren't checked if there was a dot in + filepath. + - Addition of new dwav and dlnd compsets and functionality. + - Errput may be empty string or None. + - Return aprun args but use executable string from xml. + - Add illegal char : to check_name test. + - Simplify netcdf lib args on hobart. + - Bug fix for DART in case_st_archive.py. + - Get Fortran unit tests working on cheyenne. + - Two changes to fix the Fortran unit tests that had gotten broken + recently. + - Improve create_test output by dumping case errors directly to screen. + - Fix NODEFAIL test on cheyenne. + - Fix indentation problem in case_setup. + - Allow user_mods directory in compset definition. + - Fix typo in cori-knl bind command. + - Big cs.status upgrade. + - Force user to always go through case.submit. + - Promote to warning if machine doesn't match probed machine. + - Make Machines.get_value more likely to return values of the correct + type. + - Remove conditional pio1 code. + - Fixes for PRE test. + - Set rearr opts using a pio interface. + - Bring in latest version of PIO1 to CIME. + +User interface changes: + - Moves run_tests.py to scripts/fortran_unit_testing. + - Adds PET test to cime_developer, PET behavior change. + - Output time per built component. + - config_compilers should now give MPILIB and compile_threaded + attributes for PFUNIT. + - The default mode of operation for run_tests.py is now serial. + - Other components that have unit tests (CLM and CAM) will need to + rework their top-level CMakeLists.txt file slightly. + - Do not allow ':' in testid or case name. + - Output changes for create_test. + - Significant to cs.status. + - Minor output changes to create_newcase. + - New PIO1 interface to set rearranger options. + - Signficant: new preview_run tool and new XML variables. + + +PR summary: git log --oneline --first-parent [previous_tag]..master +6d8ccb3 Merge pull request #1403 from fvitt/esmf +fe82bd6 Merge pull request #1413 from billsacks/billsacks/glc_tavg_v2 +dc29942 Merge pull request #1359 from ESMCI/jgfouca/new_query_tools +9c5b1aa Merge pull request #1428 from billsacks/move_unit_tests +c726ded Merge pull request #1435 from ESMCI/erichlf/create_newcase/tabs2spaces +8924bb3 Merge pull request #1421 from jedwards4b/hobart_nag_fix +f59dd4d Merge pull request #1414 from ESMCI/jgfouca/pet_fix_plus_coverage +67b1be7 Merge pull request #1410 from jedwards4b/executable_none_fix +0fe9e3d Merge pull request #1412 from ESMCI/jgfouca/fix_scripts_regr_test_rc +7c28d90 Merge pull request #1405 from ESMCI/jgfouca/xmlchange_append_add_space +68ab80b Merge pull request #1406 from ESMCI/jgfouca/test_sched_bugfix +da9d3b8 Merge pull request #1299 from ESMCI/fischer/testreporter_rewrite_python +f8f0fec Merge pull request #1402 from ESMCI/jgfouca/show_build_time_per_component +dc1a7ce Merge pull request #1401 from ESMCI/jgfouca/add_input_dir_to_create_newcase +962b20b Merge pull request #1396 from billsacks/compiler_from_macros +af32abd Merge pull request #1395 from ESMCI/erichlf/Build/Namelist.pm +9f5d489 Merge pull request #1394 from jedwards4b/system_test_rerun_improvement +1db632a Merge pull request #1391 from ESMCI/sarich/fix_bug_in_check_lockedfiles +5cda778 Merge pull request #1365 from ESMCI/mvertens/data_wave +2742c17 Merge pull request #1390 from ESMCI/jgfouca/fix_bug_in_errput_run_cmd +6dd85dd Merge pull request #1385 from jedwards4b/aprun_fix +4fb16a1 Merge pull request #1383 from jedwards4b/new_illegal_char +40a6017 Merge pull request #1386 from jedwards4b/hobart_netcdf_slib_fix +5a141f8 Merge pull request #1375 from ESMCI/bertinia/st_archive +06c5645 Merge pull request #1381 from billsacks/cheyenne_unit_tests +9629886 Merge pull request #1378 from jedwards4b/funit_test_fix +d79922e Merge pull request #1377 from ESMCI/jgfouca/improve_create_test_output +9d9f09c Merge pull request #1370 from ESMCI/jgfouca/minor_fix_for_cheyenne +a99eab3 Merge pull request #1373 from jedwards4b/pio_adjust_indentation_fix +ca289d9 Merge pull request #1366 from ESMCI/mvertens/usermods_in_compset +aaee38a Merge pull request #1369 from jedwards4b/cori-knlfix +d0e277d Merge pull request #1363 from ESMCI/jgfouca/big_cs_status_upgrade +dd5a30b Merge pull request #1362 from ESMCI/jgfouca/force_resubmit_to_use_submit +8282a8f Merge pull request #1361 from ESMCI/jgfouca/inform_user_of_machine +2530c66 Merge pull request #1360 from ESMCI/jgfouca/machine_get_value_type +1335a62 Merge pull request #1357 from ESMCI/jayeshkrishna/rem_cond_pio1_code +9903472 Merge pull request #1350 from gold2718/pausePOP +9f9da67 Merge pull request #1354 from ESMCI/jayeshkrishna/pio_use_set_rearr_opts +be0f9a4 Merge pull request #1352 from ESMCI/jayeshkrishna/bring_latest_pio1 + + +Modified files: git diff --name-status [previous_tag] + +M CMakeLists.txt +D README.unit_testing +M config/acme/machines/config_compilers.xml +M config/acme/machines/userdefined_laptop_template/config_compilers.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/config_machines.xml +M config/cesm/machines/userdefined_laptop_template/config_compilers.xml +M config/config_tests.xml +M config/xml_schemas/config_compsets.xsd +M config/xml_schemas/config_grids_v2.xsd +M config/xml_schemas/config_machines.xsd +A scripts/Tools/cs.status +D scripts/Tools/cs_status +A scripts/Tools/preview_run +A scripts/Tools/testreporter.py +M scripts/Tools/xmlchange +M scripts/create_newcase +M scripts/create_test +A scripts/fortran_unit_testing/Examples/circle_area/src/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/src/circle.F90 +A scripts/fortran_unit_testing/Examples/circle_area/tests/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 +A scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf +A scripts/fortran_unit_testing/Examples/interpolate_1d/src/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf +A scripts/fortran_unit_testing/Examples/test_list.xml +A scripts/fortran_unit_testing/README +A scripts/fortran_unit_testing/python/.gitignore +A scripts/fortran_unit_testing/python/printer.py +A scripts/fortran_unit_testing/python/test_xml_test_list.py +A scripts/fortran_unit_testing/python/xml_test_list.py +A scripts/fortran_unit_testing/run_tests.py +M scripts/lib/CIME/BuildTools/cmakemacroswriter.py +M scripts/lib/CIME/BuildTools/configure.py +M scripts/lib/CIME/SystemTests/nck.py +M scripts/lib/CIME/SystemTests/nodefail.py +M scripts/lib/CIME/SystemTests/pre.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/XML/compsets.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/machines.py +A scripts/lib/CIME/XML/test_reporter.py +M scripts/lib/CIME/aprun.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/case_test.py +M scripts/lib/CIME/check_lockedfiles.py +M scripts/lib/CIME/code_checker.py +M scripts/lib/CIME/nmlgen.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/test_status.py +A scripts/lib/CIME/tests/test_user_mod_support.py +M scripts/lib/CIME/user_mod_support.py +M scripts/lib/CIME/utils.py +M scripts/lib/cs.status.template +A scripts/lib/testreporter.template +M scripts/lib/update_acme_tests.py +M scripts/tests/scripts_regression_tests.py +M src/components/data_comps/datm/cime_config/config_component.xml +M src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +M src/components/data_comps/datm/datm_comp_mod.F90 +M src/components/data_comps/desp/desp_comp_mod.F90 +M src/components/data_comps/desp/esp_comp_mct.F90 +M src/components/data_comps/dice/cime_config/config_component.xml +M src/components/data_comps/dice/cime_config/namelist_definition_dice.xml +M src/components/data_comps/dice/dice_comp_mod.F90 +M src/components/data_comps/dlnd/cime_config/buildnml +M src/components/data_comps/dlnd/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +M src/components/data_comps/docn/docn_comp_mod.F90 +M src/components/data_comps/drof/cime_config/namelist_definition_drof.xml +M src/components/data_comps/drof/drof_comp_mod.F90 +M src/components/data_comps/dwav/cime_config/buildnml +M src/components/data_comps/dwav/cime_config/config_component.xml +M src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +M src/components/data_comps/dwav/dwav_comp_mod.F90 +M src/drivers/mct/cime_config/buildexe +M src/drivers/mct/cime_config/buildnml +M src/drivers/mct/cime_config/config_component.xml +M src/drivers/mct/cime_config/config_component_acme.xml +M src/drivers/mct/cime_config/config_component_cesm.xml +M src/drivers/mct/cime_config/config_compsets.xml +M src/drivers/mct/cime_config/namelist_definition_drv.xml +M src/drivers/mct/main/cesm_comp_mod.F90 +M src/drivers/mct/main/prep_glc_mod.F90 +M src/drivers/mct/main/seq_io_mod.F90 +M src/drivers/mct/main/seq_rest_mod.F90 +M src/drivers/mct/shr/seq_comm_mct.F90 +M src/drivers/mct/shr/seq_infodata_mod.F90 +M src/drivers/mct/shr/seq_timemgr_mod.F90 +M src/drivers/mct/unit_test/CMakeLists.txt +M src/drivers/mct/unit_test/stubs/CMakeLists.txt +A src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 +A src/externals/CMake/CIME_initial_setup.cmake +M src/externals/CMake/CIME_utils.cmake +M src/externals/CMake/README.md +M src/externals/CMake/pFUnit_utils.cmake +M src/externals/pio1/pio/CMakeLists.txt +M src/externals/pio1/pio/pio.F90 +M src/externals/pio1/pio/piolib_mod.F90 +M src/share/unit_test_stubs/pio/pio.F90.in +M src/share/util/shr_pio_mod.F90 +D tools/unit_testing/Examples/circle_area/src/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/src/circle.F90 +D tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 +D tools/unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf +D tools/unit_testing/Examples/interpolate_1d/src/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 +D tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 +D tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf +D tools/unit_testing/Examples/test_list.xml +D tools/unit_testing/README +D tools/unit_testing/python/.gitignore +D tools/unit_testing/python/comparable.py +D tools/unit_testing/python/environment.py +D tools/unit_testing/python/machine_setup.py +D tools/unit_testing/python/printer.py +D tools/unit_testing/python/test_environment.py +D tools/unit_testing/python/test_xml_test_list.py +D tools/unit_testing/python/test_xml_utils.py +D tools/unit_testing/python/xml_test_list.py +D tools/unit_testing/python/xml_utils.py +D tools/unit_testing/run_tests.py +M utils/perl5lib/Build/Namelist.pm +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-14-2017 +Tag: cime5.3.0-alpha.07 +Answer Changes: None +Tests on various PRs: by-hand, code_checker, pylint, scripts_regression_tests, + cime_developer with gnu compiler on cheyenne, All gnu debug tests in CESM + test suites, create B compset with ww3 at T31 resolution, + J_TestCreateNewcase, some cesm tests on cori-knl +Dependencies: + +Brief Summary: + -Updates to get going on cori-knl + -Update domain_b in nnsm map + -Fix bug where create_test was not handling user-selected projects correctly + in the case where the test_root depended on PROJECT. + -HOMME Improvement + -Unit testing: Determine machine-specific settings automatically + -Mapping/update runoff to ocn + -Fixes an issue with --user-compset flag + -Fixes for cheyenne issue + -Validate the case and test_id arguments + -Add more checks for gnu in debug mode. + -Avoid a problem with pio_typename in clone with keepexe. + -Fix blues NETCDFROOT unresolved error. + -Fix TOTAL_CORES. + -Refactor BUILD_THREADED to match documentation. + -Refactor run_cmd to handle file output streams more easily. + -More-robust way of getting path to create_test. + -Better content in TestStatus.log file. + -Improved formatting/content of TestStatus.log + -Better handling of poorly implemented batch systems. + -Fix atm grid match for T31 atm2wav map. + -Change flag for answer to create_newcase + -Fix bug in xmlquery, add regr test to catch in future + -Improve error message when user forgets --user-compset. + -Cleanup TEST_ROOT if everything works. + -Reduce newlines in CaseStatus. + +User interface changes: + -BUILD_THREADED behaves differently + -for create_newcase, --answer becomes --handle-preexisting-dirs + -Formatting changes to CaseStatus + +PR summary: git log --oneline --first-parent [previous_tag]..master +30243ac Merge pull request #1351 from jedwards4b/cori_update +1f4c4e9 Merge pull request #1349 from mnlevy1981/mapping/runoff_metadata_bugfix +7664249 Merge pull request #1348 from ESMCI/jgfouca/fix_custom_proj_test_root +6156e0a Merge pull request #1346 from ESMCI/jgfouca/homme_improvement +3779ec1 Merge pull request #1343 from billsacks/run_tests_defaults +5074c4a Merge pull request #1280 from mnlevy1981/mapping/update_runoff_to_ocn +60bfc01 Merge pull request #1341 from jedwards4b/user-compset +5e79b04 Merge pull request #1340 from jedwards4b/cheyenne_fixes +37aac0f Merge pull request #1337 from jedwards4b/validate_case_and_test_id_args +5edf3fa Merge pull request #1339 from billsacks/gnu_debug_checks +82b6181 Merge pull request #1305 from jedwards4b/clone_keepexe_and_pio_typename +4d62d80 Merge pull request #1333 from ESMCI/sarich/fix-blues-netcdfroot +12bc9b3 Merge pull request #1336 from ESMCI/jgfouca/fix_total_cores +1adf8d1 Merge pull request #1335 from ESMCI/jgfouca/cleanup_build_threaded +315e849 Merge pull request #1326 from ESMCI/jgfouca/refactor_run_cmd +997ec60 Merge pull request #1321 from ESMCI/jgfouca/cime_bisect_robustness +cb7bd10 Merge pull request #1317 from ESMCI/jgfouca/better_testlog_content +7fb258b Merge pull request #1316 from ESMCI/jgfouca/handle_titan_batch_errs +2b4f7fe Merge pull request #1320 from billsacks/t31_wav_map +7370b02 Merge pull request #1311 from ESMCI/jgfouca/better_name_for_answer +b14ebe0 Merge pull request #1312 from ESMCI/jgfouca/fix_xmlquery_listall +f741180 Merge pull request #1314 from billsacks/user_compset_errmsg +243fafa Merge pull request #1310 from ESMCI/jgfouca/better_regr_cleanup +a4c7ab7 Merge pull request #1307 from ESMCI/jgfouca/remove_useless_newlines_from_casestatus + + +Modified files: git diff --name-status [previous_tag] +M README.unit_testing +M config/acme/machines/config_machines.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_batch.xml +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/config_machines.xml +M config/cesm/machines/config_pio.xml +M config/xml_schemas/config_machines.xsd +M scripts/Tools/cime_bisect +M scripts/Tools/xmlquery +M scripts/create_clone +M scripts/create_newcase +M scripts/create_test +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/SystemTests/homme.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/SystemTests/system_tests_compare_two.py +M scripts/lib/CIME/XML/compsets.py +M scripts/lib/CIME/XML/entry_id.py +M scripts/lib/CIME/XML/env_base.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/machines.py +M scripts/lib/CIME/aprun.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/hist_utils.py +M scripts/lib/CIME/preview_namelists.py +M scripts/lib/CIME/provenance.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/utils.py +M scripts/lib/CIME/wait_for_tests.py +M scripts/tests/scripts_regression_tests.py +M src/drivers/mct/cime_config/config_component.xml +M src/externals/pio1/timing/COPYING +M src/externals/pio2/src/gptl/COPYING +M src/share/timing/COPYING +M src/share/util/mct_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL +M tools/mapping/gen_mapping_files/runoff_to_ocn/README +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore +A tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl +A tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_r05_gx3v7.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_rx1_gx1v6.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_tx01.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_wr50a_ar9v4.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_wr50a_gx3v7.nml +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 +M tools/unit_testing/run_tests.py + +====================================================================== + Originator:Chris Fischer Date: 4-4-2017 Tag: cime5.3.0-alpha.06 diff --git a/LICENSE.TXT b/LICENSE.TXT index d832be2be936..6d0ae1a9a2e4 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,9 +1,15 @@ Copyright (c) 2015, University Corporation for Atmospheric Research (UCAR) All rights reserved. + and +Copyright (c) 2017, Sandia Corporation. +Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +the U.S. Government retains certain rights in this software. Developed by: University Corporation for Atmospheric Research - National Center for Atmospheric Research https://www2.cesm.ucar.edu/working-groups/sewg + and + DOE BER ACME project team members, including those at SNL and ANL Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -17,7 +23,7 @@ the Software is furnished to do so, subject to the following conditions: - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimers in the documentation and/or other materials provided with the distribution. - - Neither the names of [Name of Development Group, UCAR], + - Neither the names of UCAR or Sandia Corporation, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission. diff --git a/README b/README deleted file mode 100644 index ec161702495d..000000000000 --- a/README +++ /dev/null @@ -1,31 +0,0 @@ -The cime directory structure (other than the externals/ directory) -was generated from the following trunk tags: - -cime/driver_cpl https://svn-ccsm-models.cgd.ucar.edu/drv/seq_mct/trunk_tags/drvseq5_1_15 - -cime/components/data_comps/datm https://svn-ccsm-models.cgd.ucar.edu/datm7/trunk_tags/datm8_150310 -cime/components/data_comps/dice https://svn-ccsm-models.cgd.ucar.edu/dice7/trunk_tags/dice8_150310 -cime/components/data_comps/dlnd https://svn-ccsm-models.cgd.ucar.edu/dlnd7/trunk_tags/dlnd8_150310 -cime/components/data_comps/docn https://svn-ccsm-models.cgd.ucar.edu/docn7/trunk_tags/docn8_150310 -cime/components/data_comps/drof https://svn-ccsm-models.cgd.ucar.edu/drof/trunk_tags/drof_150310 -cime/components/stub_comps https://svn-ccsm-models.cgd.ucar.edu/stubs/trunk_tags/stubs1_4_08 -cime/components/xcpl_comps https://svn-ccsm-models.cgd.ucar.edu/dead7/trunk_tags/dead7_8_04 - -cime/machines https://svn-ccsm-models.cgd.ucar.edu/Machines/trunk_tags/Machines_150309 - -cime/scripts https://svn-ccsm-models.cgd.ucar.edu/scripts/trunk_tags/scripts_150309 - -cime/share/csm_share https://svn-ccsm-models.cgd.ucar.edu/csm_share/trunk_tags/share3_150116 -cime/share/esmf_wrf_timemgr https://svn-ccsm-models.cgd.ucar.edu/esmf_wrf_timemgr/trunk_tags/esmf_wrf_timemgr_141217 -cime/share/timing https://svn-ccsm-models.cgd.ucar.edu/timing/trunk_tags/timing_150302 - -cime/utils/pythonlib https://svn-ccsm-models.cgd.ucar.edu/scripts/trunk_tags/scripts4_150204/scripts/ccsm_utils/Tools/pythonlib -cime/utils/perl5lib https://svn-ccsm-models.cgd.ucar.edu/perl5lib/trunk_tags/perl5lib_150302 - -cime/tools/load_balancing_tool https://svn-ccsm-models.cgd.ucar.edu/tools/load_balancing_tool/trunk_tags/load_balancing_tool_141008 -cime/tools/unit_testing https://svn-ccsm-models.cgd.ucar.edu/unit_testing/trunk_tags/unit_testing_0_16 -cime/tools/statistical_ensemble_test https://svn-ccsm-models.cgd.ucar.edu/validation_testing/trunk_tags/validation_20140708/run_CESM/ -cime/tools/mapping https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106 -cime/tools/cprnc https://svn-ccsm-models.cgd.ucar.edu/tools/cprnc/trunk_tags/cprnc_150301 - - diff --git a/README.md b/README.md index 73e19ea18d85..05a643e2f477 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,16 @@ Common Infrastructure for Modeling the Earth **cime** (pronounced: seem) represents the infrastructure code for the Community Earth System Model - (CESM) and the Accelerated Climate Model for Energy (ACME). -*cime* includes scripts for configuration, build, and testing of -models, as well as data and stub models for climate components, -and a code for coupling different climate components together. - + (CESM) and the +Accelerated Climate Model for Energy (ACME). +*cime* providess scripts for configuration, build, and testing of +models, as well as code for data and stub models for climate components, +and a driver code for bringing all the climate components together in a single executable. + +# Documentation + +See esmci.github.io/cime + # Developers *cime* was initially extracted from CESM as a stand-alone capability in 2015 @@ -28,18 +33,12 @@ Mariana Vertenstein | NCAR | 1 - D | NSF, DOE Jim Edwards | NCAR | 1 - D | NSF (1-D), DOE(1-2) Jim Foucar | SNL | 5 - D | DOE Rob Jacob | ANL | 5 - D | DOE +Bill Sacks | NCAR | 1 - D | NSF Andreas Wilke | ANL | 5 - D | DOE Jason Sarich | ANL | 5 - D | DOE -??Sean Santos | NCAR | 1 - 4 | NSF +Michael Deakin | SNL | 5 - D | DOE +Erich Foster | SNL | 5 - D | DOE +Sean Santos | NCAR | 1 - 4 | NSF _Key: D = Current development version (i.e. still active on project)_ -# Documentation - -Currently, this is a work in progress. - - - - - - diff --git a/README.unit_testing b/README.unit_testing deleted file mode 100644 index df92424f99df..000000000000 --- a/README.unit_testing +++ /dev/null @@ -1,16 +0,0 @@ -# To run all CIME unit tests on caldera, run the following command: -# (Note that this must be done from an interactive caldera session, not from yellowstone) -# Note also that this requires module load all-python-libs -# -# The creation of a temporary directory ensures that you are doing a completely -# clean build of the unit tests. (The use of the --clean flag to run_tests.py -# cleans most, but not all of the files created by the unit test build.) For -# rerunning the tests after an incremental change, you can instead use an -# existing build directory. -# -# We would encourage you to port these tests to other platforms. -# The test requires an install of pFunit available from -# https://sourceforge.net/projects/pfunit/ -# - -tools/unit_testing/run_tests.py --build-dir `mktemp -d ./unit_tests.XXXXXXXX` diff --git a/config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml index a6edbfdcc13d..aaefcf270e05 100644 --- a/config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -158,7 +158,7 @@ -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} - shared + shared batch @@ -177,8 +177,8 @@ + debug regular - debug @@ -198,8 +198,8 @@ --constraint=haswell + debug regular - debug @@ -208,8 +208,8 @@ --constraint=knl,quad,cache + debug regular - debug @@ -361,7 +361,7 @@ batch - debug + debug diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml index 513116d747d0..a059df4130c7 100644 --- a/config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -807,7 +807,6 @@ for mct, etc. -lmkl_core -lmkl_gnu_thread -lmkl_blacs_openmpi_lp64 \ -Wl,--end-group -lgomp -lrt - @@ -916,7 +915,6 @@ for mct, etc. mpixlf77_r /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/ /soft/libraries/pnetcdf/1.6.0/cnk-xl/current/ - /home/santos/pFUnit/pFUnit_IBM /soft/libraries/hdf5/1.8.14/cnk-xl/current/ -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp @@ -939,7 +937,6 @@ for mct, etc. mpixlf77_r /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/ /soft/libraries/pnetcdf/1.6.0/cnk-xl/current/ - /home/santos/pFUnit/pFUnit_IBM /soft/libraries/hdf5/1.8.14/cnk-xl/current/ -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 9138c818d0ad..733702d49d53 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -604,7 +604,7 @@ sems-python/2.7.9 - sems-gcc/5.1.0 + sems-gcc/5.3.0 sems-intel/15.0.2 diff --git a/config/acme/machines/template.case.run b/config/acme/machines/template.case.run index ea81a1585279..56d425e34013 100755 --- a/config/acme/machines/template.case.run +++ b/config/acme/machines/template.case.run @@ -55,9 +55,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", help="Case directory to build") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/acme/machines/template.case.test b/config/acme/machines/template.case.test index 1c2cf6e49b05..5fbb6b5c35f0 100755 --- a/config/acme/machines/template.case.test +++ b/config/acme/machines/template.case.test @@ -50,9 +50,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", help="Case directory to build") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/acme/machines/template.lt_archive b/config/acme/machines/template.lt_archive index 421cca9325af..0feb5f9d3777 100755 --- a/config/acme/machines/template.lt_archive +++ b/config/acme/machines/template.lt_archive @@ -49,9 +49,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", default=os.getcwd(), help="Case directory to build") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/acme/machines/userdefined_laptop_template/config_compilers.xml b/config/acme/machines/userdefined_laptop_template/config_compilers.xml index 03970f6a16b0..f3ecb9dce845 100644 --- a/config/acme/machines/userdefined_laptop_template/config_compilers.xml +++ b/config/acme/machines/userdefined_laptop_template/config_compilers.xml @@ -29,7 +29,7 @@ TRUE /usr/local $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate - $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e + $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e @@ -57,6 +57,6 @@ TRUE /opt/local $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate - $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e + $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e diff --git a/config/cesm/config_grids.xml b/config/cesm/config_grids.xml index 269839e58c7a..ab022a0580bb 100644 --- a/config/cesm/config_grids.xml +++ b/config/cesm/config_grids.xml @@ -361,13 +361,20 @@ gx1v7 - + 0.9x1.25 0.9x1.25 0.9x1.25 gx1v6 + + 0.9x1.25 + 0.9x1.25 + 0.9x1.25 + null + + 0.9x1.25 0.9x1.25 @@ -506,6 +513,13 @@ gx1v6 + + 1.9x2.5 + 1.9x2.5 + 1.9x2.5 + null + + 1.9x2.5 1.9x2.5 @@ -848,10 +862,9 @@ gx1v7 - - ww3a - ww3a - ww3a + + + ww3a @@ -959,9 +972,11 @@ 288 192 domain.lnd.fv0.9x1.25_gx1v6.090309.nc - domain.ocn.0.9x1.25_gx1v6_090403.nc + domain.ocn.0.9x1.25_gx1v6_090403.nc domain.lnd.fv0.9x1.25_gx1v7.151020.nc - domain.ocn.fv0.9x1.25_gx1v7.151020.nc + domain.ocn.fv0.9x1.25_gx1v7.151020.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc + /glade/u/home/benedict/ys/datain/domain.aqua.fv0.9x1.25.nc 0.9x1.25 is FV 1-deg grid: @@ -969,6 +984,7 @@ 144 96 domain.lnd.fv1.9x2.5_gx1v6.090206.nc domain.ocn.1.9x2.5_gx1v6_090403.nc + domain.aqua.fv1.9x2.5.nc 1.9x2.5 is FV 2-deg grid: @@ -1483,7 +1499,7 @@ cpl/gridmaps/gx1v7/map_gx1v7_TO_ww3a_splice_170214.nc - + cpl/gridmaps/T31/map_T31_TO_ww3a_bilin_131104.nc @@ -1612,16 +1628,16 @@ - cpl/gridmaps/rx1/map_rx1_to_gx3v7_e1000r500_161214.nc - cpl/gridmaps/rx1/map_rx1_to_gx3v7_e1000r500_161214.nc + cpl/gridmaps/rx1/map_rx1_to_gx3v7_nnsm_e1000r500_170413.nc + cpl/gridmaps/rx1/map_rx1_to_gx3v7_nnsm_e1000r500_170413.nc cpl/gridmaps/rx1/map_rx1_to_gx1v6_nn_ac_161213.nc cpl/gridmaps/rx1/map_rx1_to_gx1v6_e1000r300_161212.nc - cpl/gridmaps/rx1/map_rx1_to_gx1v7_nn_ac_161213.nc - cpl/gridmaps/rx1/map_rx1_to_gx1v7_e1000r300_161213.nc + cpl/gridmaps/rx1/map_rx1_to_gx1v7_nn_open_ocean_nnsm_e1000r300_marginal_sea_170413.nc + cpl/gridmaps/rx1/map_rx1_to_gx1v7_nnsm_e1000r300_170413.nc cpl/gridmaps/rx1/map_rx1_to_tx1v1_e1000r300_161214.nc @@ -1637,16 +1653,16 @@ - cpl/gridmaps/r05/map_r05_to_gx3v7_e1000r500_161214.nc - cpl/gridmaps/r05/map_r05_to_gx3v7_e1000r500_161214.nc + cpl/gridmaps/r05/map_r05_to_gx3v7_nnsm_e1000r500_170413.nc + cpl/gridmaps/r05/map_r05_to_gx3v7_nnsm_e1000r500_170413.nc cpl/gridmaps/r05/map_r05_to_gx1v6_nn_ac_161214.nc cpl/gridmaps/r05/map_r05_to_gx1v6_e1000r300_161212.nc - cpl/gridmaps/r05/map_r05_to_gx1v7_nn_ac_161213.nc - cpl/gridmaps/r05/map_r05_to_gx1v7_e1000r300_161213.nc + cpl/gridmaps/r05/map_r05_to_gx1v7_nn_open_ocean_nnsm_e1000r300_marginal_sea_170413.nc + cpl/gridmaps/r05/map_r05_to_gx1v7_nnsm_e1000r300_170413.nc cpl/gridmaps/r05/map_r05_to_tx1v1_e1000r500_161214.nc diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml index 419cd046ecbc..a1008a91d0dc 100644 --- a/config/cesm/machines/config_batch.xml +++ b/config/cesm/machines/config_batch.xml @@ -383,7 +383,7 @@ sbatch - -C knl + -C knl,quad,cache regular diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index f9890da41c4f..0bfcde185360 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -283,7 +283,6 @@ using a fortran linker. -C=all -g -time -f2003 -ieee=stop -gline - -openmp -g -time -f2003 -ieee=stop -gline - -openmp -fixed @@ -307,9 +305,6 @@ using a fortran linker. -free FALSE - - -openmp - mpicc mpif90 gcc @@ -549,6 +544,8 @@ using a fortran linker. -mkl + + FALSE @@ -566,11 +563,13 @@ using a fortran linker. -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf - -DHAVE_PAPI -DHAVE_SLASHPROC + -DHAVE_SLASHPROC -mkl -lmemkind -zmuldefs + + FALSE @@ -663,7 +662,7 @@ using a fortran linker. mpich $ENV{NETCDF_PATH} - $SHELL{${NETCDF_PATH}/bin/nf-config --flibs} + -L$NETCDF_PATH/lib -lnetcdff -lnetcdf @@ -683,10 +682,10 @@ using a fortran linker. -Wl,-rpath,$ENV{MPI_PATH}/lib -lifcore - /home/santos/pFUnit/pFUnit_Intel_3_0 -mkl=cluster + /fs/cgd/csm/tools/pFUnit/pFUnit3.2.8_hobart_Intel15.0.2_noMPI_noOpenMP @@ -697,7 +696,9 @@ using a fortran linker. -lpthread - /home/santos/pFUnit/pFUnit_NAG_3_0 + + -L/usr/local/nag/lib/NAG_Fortran + @@ -763,7 +764,6 @@ using a fortran linker. /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlf2003_r /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/ - /home/santos/pFUnit/pFUnit_IBM gpfs /soft/libraries/pnetcdf/1.6.1/cnk-xl/current/ /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r @@ -926,7 +926,6 @@ using a fortran linker. mpixlc_r mpixlf2003_r /soft/libraries/netcdf/4.3.0-f4.2/cnk-xl/V1R2M0-20131211/ - /home/santos/pFUnit/pFUnit_IBM gpfs /soft/libraries/pnetcdf/1.3.1/cnk-xl/current/ mpixlc_r @@ -988,6 +987,8 @@ using a fortran linker. -DPIO_ENABLE_LOGGING=ON + $ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_noMPI_noOpenMP + $ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_MPI_openMP FALSE @@ -1032,7 +1033,8 @@ using a fortran linker. mpiicpc /glade/apps/opt/papi/5.3.0/intel/12.1.5/include/ /glade/apps/opt/papi/5.3.0/intel/12.1.5/lib64 - $ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI + $ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_yellowstone_Intel15.0.1_noMPI_noOpenMP + $ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI icc ifort diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 276e0cae774f..d8992f137175 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -219,10 +219,18 @@ mpiexec_mpt + -np $TOTALPES -p "%g:" - omplace + + omplace + + + /opt/sgi/mpt/mpt-2.15/bin/mpirun $ENV{UNIT_TEST_HOST} -np 1 + @@ -242,6 +250,19 @@ intel/17.0.1 mkl + esmf_libs + + + esmf-7.0.0-defio-mpi-g + + + esmf-7.0.0-defio-mpi-O + + + esmf-7.0.0-ncdfio-uni-g + + + esmf-7.0.0-ncdfio-uni-O gnu/6.3.0 @@ -262,6 +283,12 @@ 256M 16 + + + false + @@ -432,7 +459,7 @@ PrgEnv-cray - cce cce/8.5.4 + cce cce/8.5.7 PrgEnv-gnu @@ -491,6 +518,7 @@ --label -n $TOTALPES + -c 4 --cpu_bind=cores @@ -535,7 +563,7 @@ PrgEnv-cray - cce cce/8.5.4 + cce cce/8.5.7 PrgEnv-gnu @@ -544,7 +572,7 @@ cray-memkind craype-mic-knl - papi/5.4.3.2 + craype craype/2.5.7 @@ -1679,16 +1707,16 @@ esmf - esmf-6.3.0rp1-defio-mpi-g + esmf-7.0.0-ncdfio-mpi-g - esmf-6.3.0rp1-defio-mpi-O + esmf-7.0.0-defio-mpi-O - esmf-6.3.0rp1-ncdfio-uni-g + esmf-7.0.0-ncdfio-uni-g - esmf-6.3.0rp1-ncdfio-uni-O + esmf-7.0.0-ncdfio-uni-O pgi/15.10 diff --git a/config/cesm/machines/template.case.run b/config/cesm/machines/template.case.run index 24162fe45951..7f9d31fc5f59 100755 --- a/config/cesm/machines/template.case.run +++ b/config/cesm/machines/template.case.run @@ -52,9 +52,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", help="Case directory to build") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/cesm/machines/template.case.test b/config/cesm/machines/template.case.test index e918de6ec21f..13dbb65a4ce7 100755 --- a/config/cesm/machines/template.case.test +++ b/config/cesm/machines/template.case.test @@ -46,9 +46,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", help="Case directory to build") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/cesm/machines/template.lt_archive b/config/cesm/machines/template.lt_archive index 1dd9de1a6bc0..9f51cfa886fe 100755 --- a/config/cesm/machines/template.lt_archive +++ b/config/cesm/machines/template.lt_archive @@ -47,9 +47,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--caseroot", default=os.getcwd(), help="Case directory to build") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.caseroot is not None: os.chdir(args.caseroot) diff --git a/config/cesm/machines/userdefined_laptop_template/config_compilers.xml b/config/cesm/machines/userdefined_laptop_template/config_compilers.xml index 03970f6a16b0..f3ecb9dce845 100644 --- a/config/cesm/machines/userdefined_laptop_template/config_compilers.xml +++ b/config/cesm/machines/userdefined_laptop_template/config_compilers.xml @@ -29,7 +29,7 @@ TRUE /usr/local $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate - $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e + $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e @@ -57,6 +57,6 @@ TRUE /opt/local $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate - $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e + $ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e diff --git a/config/config_tests.xml b/config/config_tests.xml index e078b924e40e..6bef57899c77 100644 --- a/config/config_tests.xml +++ b/config/config_tests.xml @@ -158,12 +158,53 @@ LAR long term archive test DAE data assimilation test: non answer changing +PRE pause-resume test: by default a BFB test of pause-resume cycling + ====================================================================== Other component-specific tests ====================================================================== LII CLM initial condition interpolation test +====================================================================== + Infrastructural tests for CIME. These are used by scripts_regression_tests. + Users won't generally run these. +====================================================================== + + +TESTBUILDFAIL Insta-fail build step. Used to confirm that failed + builds are caught and reported correctly. + +TESTBUILDFAILEXC Insta-fail build step by failing to init. Used to test + correct behavior when exceptions are generated. + +TESTRUNFAIL Insta-fail run step. Used to confirm that model run + failures are caught and reported correctly. + +TESTRUNFAILEXC Insta-fail run step via exception. Used to test correct + correct behavior when exceptions are generated. + +TESTRUNPASS Insta-pass run step. Used to test that run that work + are reported correctly. + +TESTMEMLEAKFAIL Insta-fail memleak step. Used to test that memleaks are + detected and reported correctly. + +TESTMEMLEAKPASS Insta-pass memleak step. Used to test that non-memleaks are + reported correctly. + +TESTRUNDIFF Produces a canned hist file. Env var TESTRUNDIFF_ALTERNATE can + be used to cause a DIFF. Used to check that baseline diffs are + detected and reported correctly. + +TESTTESTDIFF Simulates internal test diff (non baseline). Used to check that + internal comparison failures are detected and reported correctly. + +TESTRUNSLOWPASS After 5 minutes of sleep, pass run step. Used to test timeouts + and kills. + +NODEFAIL Tests restart upon detected node failure. Generates fake failures, + the number of which is controlled by NODEFAIL_NUM_FAILS. --> @@ -378,7 +419,8 @@ LII CLM initial condition interpolation test For testing infra only. Tests restart upon detected node failure 1 - ndays + nsteps + $ATM_NCPL 11 $STOP_N / 2 + 1 $STOP_OPTION diff --git a/config/xml_schemas/config_batch.xsd b/config/xml_schemas/config_batch.xsd index da8e028a5b38..ad810c028708 100644 --- a/config/xml_schemas/config_batch.xsd +++ b/config/xml_schemas/config_batch.xsd @@ -69,7 +69,7 @@ @@ -130,6 +130,7 @@ + diff --git a/config/xml_schemas/config_compsets.xsd b/config/xml_schemas/config_compsets.xsd index e9072738a6d7..e2b64df2c666 100644 --- a/config/xml_schemas/config_compsets.xsd +++ b/config/xml_schemas/config_compsets.xsd @@ -11,6 +11,7 @@ + @@ -32,6 +33,7 @@ + diff --git a/config/xml_schemas/config_grids_v2.xsd b/config/xml_schemas/config_grids_v2.xsd index 2bac0a1de037..b4680b93f9e3 100644 --- a/config/xml_schemas/config_grids_v2.xsd +++ b/config/xml_schemas/config_grids_v2.xsd @@ -4,7 +4,7 @@ - + diff --git a/config/xml_schemas/config_machines.xsd b/config/xml_schemas/config_machines.xsd index ee31239c4c04..bad0fb386dc6 100644 --- a/config/xml_schemas/config_machines.xsd +++ b/config/xml_schemas/config_machines.xsd @@ -188,6 +188,7 @@ + diff --git a/scripts/Tools/acme_check_env b/scripts/Tools/acme_check_env index 7abdbcf71fad..e6edf90576c4 100755 --- a/scripts/Tools/acme_check_env +++ b/scripts/Tools/acme_check_env @@ -30,9 +30,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter CIME.utils.setup_standard_logging_options(parser) - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) ############################################################################### def check_sh(): diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results index 4b01e4768f8c..ceca60cf51f2 100755 --- a/scripts/Tools/bless_test_results +++ b/scripts/Tools/bless_test_results @@ -89,9 +89,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("bless_tests", nargs="*", help="When blessing, limit the bless to tests matching these regex") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) expect(not (args.report_only and args.force), "Makes no sense to use -r and -f simultaneously") diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 7fba3c18d617..8caee2ee579a 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -56,9 +56,7 @@ OR parser.add_argument("--clean-all", action="store_true", help="clean all objects including sharedlibobjects that may be used by other builds") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) cleanlist = args.clean if args.clean is None or len(args.clean) else comps diff --git a/scripts/Tools/case.cmpgen_namelists b/scripts/Tools/case.cmpgen_namelists index e874121a205c..999ba075a45c 100755 --- a/scripts/Tools/case.cmpgen_namelists +++ b/scripts/Tools/case.cmpgen_namelists @@ -49,9 +49,7 @@ OR help="Force generation to use baselines with this name. " "Default will be to follow the case specification") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.caseroot, args.compare, args.generate, args.compare_name, args.generate_name diff --git a/scripts/Tools/case.setup b/scripts/Tools/case.setup index ea60b98b1daa..7fb4eaae6daf 100755 --- a/scripts/Tools/case.setup +++ b/scripts/Tools/case.setup @@ -45,16 +45,9 @@ OR parser.add_argument("-r", "--reset", action="store_true", help="Does a clean followed by setup") - parser.add_argument("--no-adjust-pio", action="store_true", - help="Do NOT adjust pio settings for new pelayout." - "By default if the pelayout is changed the pio layout will be adjusted." - "This option overrides that feature and leaves the pio layout as currently set.") + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) - - return args.caseroot, args.clean, args.test_mode, args.reset, not args.no_adjust_pio + return args.caseroot, args.clean, args.test_mode, args.reset ############################################################################### def _main_func(description): @@ -63,9 +56,9 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - caseroot, clean, test_mode, reset, adjust_pio = parse_command_line(sys.argv, description) + caseroot, clean, test_mode, reset = parse_command_line(sys.argv, description) with Case(caseroot, read_only=False) as case: - case_setup(case, clean=clean, test_mode=test_mode, reset=reset, adjust_pio=adjust_pio) + case_setup(case, clean=clean, test_mode=test_mode, reset=reset) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index c307035dc43c..3ea9d21b2874 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -48,12 +48,10 @@ OR parser.add_argument("-a", "--batch-args", help="Used to pass additional arguments to batch system. ") - args = parser.parse_args(args[1:]) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) CIME.utils.expect(args.prereq is None, "--prereq not currently supported") - CIME.utils.handle_standard_logging_options(args) - return args.caseroot, args.job, args.no_batch, args.resubmit, args.batch_args ############################################################################### diff --git a/scripts/Tools/case_diff b/scripts/Tools/case_diff index 6cd5db1b7069..4ad7da8da800 100755 --- a/scripts/Tools/case_diff +++ b/scripts/Tools/case_diff @@ -42,9 +42,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-b", "--show-binary", action="store_true", help="Show binary diffs") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.case1, args.case2, args.show_binary, args.skip_list diff --git a/scripts/Tools/check_case b/scripts/Tools/check_case index 2553e1a197fc..0c360e41e879 100755 --- a/scripts/Tools/check_case +++ b/scripts/Tools/check_case @@ -38,9 +38,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter CIME.utils.setup_standard_logging_options(parser) - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) ############################################################################### def _main_func(description): @@ -68,4 +66,3 @@ def _main_func(description): if __name__ == "__main__": _main_func(__doc__) - diff --git a/scripts/Tools/check_input_data b/scripts/Tools/check_input_data index 7795c063cac1..dc1277204909 100755 --- a/scripts/Tools/check_input_data +++ b/scripts/Tools/check_input_data @@ -52,9 +52,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--download", action="store_true", help="Attempt to download missing input files") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.svn_loc, args.input_data_root, args.data_list_dir, args.download @@ -78,4 +76,3 @@ def _main_func(description): if (__name__ == "__main__"): _main_func(__doc__) - diff --git a/scripts/Tools/check_lockedfiles b/scripts/Tools/check_lockedfiles index 39b68a1bd907..46f509e4721d 100755 --- a/scripts/Tools/check_lockedfiles +++ b/scripts/Tools/check_lockedfiles @@ -29,9 +29,7 @@ OR parser.add_argument("--caseroot", default=os.getcwd(), help="Case directory to build") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.caseroot diff --git a/scripts/Tools/cime_bisect b/scripts/Tools/cime_bisect index 6e2b37e93bd8..53d1d8304129 100755 --- a/scripts/Tools/cime_bisect +++ b/scripts/Tools/cime_bisect @@ -74,9 +74,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-a", "--all-commits", action="store_true", help="Test all commits, not just merges") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if (args.test_root is None): args.test_root = os.path.join(_MACHINE.get_value("CIME_OUTPUT_ROOT"), "cime_bisect") diff --git a/scripts/Tools/code_checker b/scripts/Tools/code_checker index a81908a99512..79191d2d2bfd 100755 --- a/scripts/Tools/code_checker +++ b/scripts/Tools/code_checker @@ -46,9 +46,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("files", nargs="*", help="Restrict checking to specific files. Relative name is fine.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.num_procs, args.files diff --git a/scripts/Tools/compare_namelists b/scripts/Tools/compare_namelists index 69f86659afc2..fcfbc481a2c3 100755 --- a/scripts/Tools/compare_namelists +++ b/scripts/Tools/compare_namelists @@ -40,9 +40,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-c", "--case", action="store", dest="case", default=None, help="The case base id (..). Helps us normalize data.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) # Normalize case if (args.case is not None): diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index ae3ac037b142..7307c74e0c8f 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -90,9 +90,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("compare_tests", nargs="*", help="When comparing, limit the comparison to tests matching these regex") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.compare_tests, args.namelists_only, args.hist_only diff --git a/scripts/Tools/component_compare_baseline b/scripts/Tools/component_compare_baseline index ea8e43343a7c..a809e12acfe3 100755 --- a/scripts/Tools/component_compare_baseline +++ b/scripts/Tools/component_compare_baseline @@ -35,9 +35,7 @@ OR parser.add_argument("-b", "--baseline-dir", help="Use custom baseline dir") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.caseroot, args.baseline_dir diff --git a/scripts/Tools/component_compare_copy b/scripts/Tools/component_compare_copy index 209358ff6047..db3574cad57b 100755 --- a/scripts/Tools/component_compare_copy +++ b/scripts/Tools/component_compare_copy @@ -36,9 +36,7 @@ OR parser.add_argument("caseroot", nargs="?", default=os.getcwd(), help="Case directory") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.suffix, args.caseroot diff --git a/scripts/Tools/component_compare_test b/scripts/Tools/component_compare_test index aae11911e49c..07e6c17b199e 100755 --- a/scripts/Tools/component_compare_test +++ b/scripts/Tools/component_compare_test @@ -38,9 +38,7 @@ OR parser.add_argument("caseroot", nargs="?", default=os.getcwd(), help="Case directory") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.suffix1, args.suffix2, args.caseroot diff --git a/scripts/Tools/component_generate_baseline b/scripts/Tools/component_generate_baseline index 0dcca7681c8b..311537438dfa 100755 --- a/scripts/Tools/component_generate_baseline +++ b/scripts/Tools/component_generate_baseline @@ -40,9 +40,7 @@ OR "will raise an error. Specifying this option allows " "existing baseline directories to be silently overwritten.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.caseroot, args.baseline_dir, args.allow_baseline_overwrite diff --git a/scripts/Tools/cs.status b/scripts/Tools/cs.status new file mode 100755 index 000000000000..8250a3c09684 --- /dev/null +++ b/scripts/Tools/cs.status @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +List test results based on TestStatus files. Returns True if +no errors occured (not based on test statuses). +""" + +from standard_script_setup import * +import argparse, sys, os, logging, glob +from CIME.test_status import * + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( +usage="""\n%s [ ...] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Wait for all tests in a test area\033[0m + > %s path/to/testarea/*/TestStatus +""" % ((os.path.basename(args[0]), ) * 4), + +description=description, + +formatter_class=argparse.ArgumentDefaultsHelpFormatter +) + + parser.add_argument("paths", nargs="*", help="Paths to TestStatus files.") + + parser.add_argument("-s", "--summary", action="store_true", + help="Only show summary") + + parser.add_argument("-t", "--test-id", action="append", default=[], + help="Only show summary") + + parser.add_argument("-r", "--test-root", default=os.getcwd(), + help="Only show summary") + + args = parser.parse_args(args[1:]) + + return args.paths, args.summary, args.test_id, args.test_root + +############################################################################### +def cs_status(test_paths, summary=False): +############################################################################### + test_id_output = {} + for test_path in test_paths: + test_dir=os.path.dirname(test_path) + ts = TestStatus(test_dir=test_dir) + test_id = os.path.basename(test_dir).split(".")[-1] + test_name = ts.get_name() + status = ts.get_overall_test_status() + if not summary: + output = " %s (Overall: %s) details:\n" % (test_name, status) + output += ts.phase_statuses_dump(prefix=" ") + else: + output = " %s %s\n" % (status, test_name) + + if test_id in test_id_output: + test_id_output[test_id] += output + else: + test_id_output[test_id] = output + + for test_id in sorted(test_id_output): + print test_id + print test_id_output[test_id], + +############################################################################### +def _main_func(description): +############################################################################### + test_paths, summary, test_ids, test_root = parse_command_line(sys.argv, description) + for test_id in test_ids: + test_paths.extend(glob.glob(os.path.join(test_root, "*%s/TestStatus" % test_id))) + + cs_status(test_paths, summary) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/scripts/Tools/cs_status b/scripts/Tools/cs_status deleted file mode 100755 index fd521f7dcbf0..000000000000 --- a/scripts/Tools/cs_status +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -""" -List test results based on TestStatus files. Returns True if -no errors occured (not based on test statuses). -""" - -from standard_script_setup import * -import argparse, sys, os, logging -from CIME.test_status import * - -############################################################################### -def parse_command_line(args, description): -############################################################################### - parser = argparse.ArgumentParser( -usage="""\n%s [ ...] [--verbose] -OR -%s --help -OR -%s --test - -\033[1mEXAMPLES:\033[0m - \033[1;32m# Wait for all tests in a test area\033[0m - > %s path/to/testarea/*/TestStatus -""" % ((os.path.basename(args[0]), ) * 4), - -description=description, - -formatter_class=argparse.ArgumentDefaultsHelpFormatter -) - - CIME.utils.setup_standard_logging_options(parser) - - parser.add_argument("paths", nargs="+", help="Paths to TestStatus files.") - - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) - - return args.paths - -############################################################################### -def cs_status(test_paths): -############################################################################### - for test_path in test_paths: - ts = TestStatus(test_dir=os.path.dirname(test_path)) - test_name = ts.get_name() - summary = ts.get_overall_test_status() - print "%s (Overall: %s), details:" % (test_name, summary) - ts.phase_statuses_dump(sys.stdout, prefix=' ') - -############################################################################### -def _main_func(description): -############################################################################### - test_paths = parse_command_line(sys.argv, description) - cs_status(test_paths) - -############################################################################### - -if (__name__ == "__main__"): - _main_func(__doc__) diff --git a/scripts/Tools/getTiming b/scripts/Tools/getTiming index a2ba123ec3b4..22878582ee51 100755 --- a/scripts/Tools/getTiming +++ b/scripts/Tools/getTiming @@ -21,8 +21,8 @@ def parse_command_line(args, description): default="999999-999999") parser.add_argument("--caseroot", default=os.getcwd(), help="Case directory to get timing for") - args = parser.parse_args(args[1:]) - CIME.utils.handle_standard_logging_options(args) + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.caseroot, args.lid def __main_func(description): diff --git a/scripts/Tools/jenkins_generic_job b/scripts/Tools/jenkins_generic_job index 074614b61aa9..5ae82efe5d50 100755 --- a/scripts/Tools/jenkins_generic_job +++ b/scripts/Tools/jenkins_generic_job @@ -91,9 +91,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("--compiler", help="Compiler to use to build cime. Default will be the default defined for the machine.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) expect(not (args.submit_to_cdash and args.generate_baselines), "Does not make sense to use --generate-baselines and --submit-to-cdash together") diff --git a/scripts/Tools/list_acme_tests b/scripts/Tools/list_acme_tests index 915f0a380e1b..c7f2cefba512 100755 --- a/scripts/Tools/list_acme_tests +++ b/scripts/Tools/list_acme_tests @@ -43,9 +43,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("categories", nargs="*", help="The test categories to list. Default will list all. Test categories: %s" % (", ".join(update_acme_tests.get_test_suites()))) - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if (not args.categories): args.categories = update_acme_tests.get_test_suites() diff --git a/scripts/Tools/normalize_cases b/scripts/Tools/normalize_cases index 1eca9367d5df..693c4861e67d 100755 --- a/scripts/Tools/normalize_cases +++ b/scripts/Tools/normalize_cases @@ -38,9 +38,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("case2", help="Second case. This one will not be changed") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.case1, args.case2 diff --git a/scripts/Tools/pelayout b/scripts/Tools/pelayout index 64848d3a64e6..50736a528322 100755 --- a/scripts/Tools/pelayout +++ b/scripts/Tools/pelayout @@ -68,9 +68,7 @@ def parse_command_line(args): parser.add_argument("-caseroot" , "--caseroot", default=os.getcwd(), help="Case directory to reference") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if (args.no_header): args.header = None diff --git a/scripts/Tools/preview_namelists b/scripts/Tools/preview_namelists index a87a52191909..4bece0ee573b 100755 --- a/scripts/Tools/preview_namelists +++ b/scripts/Tools/preview_namelists @@ -17,48 +17,32 @@ import argparse, doctest ############################################################################### def parse_command_line(args, description): ############################################################################### - parser = argparse.ArgumentParser( - usage="""\n%s [--debug] -OR -%s --verbose -OR -%s --help -OR -%s --test - -\033[1mEXAMPLES:\033[0m - \033[1;32m# Run \033[0m - > %s -""" % ((os.path.basename(args[0]), ) * 5), - -description=description, - -formatter_class=argparse.ArgumentDefaultsHelpFormatter -) - + parser = argparse.ArgumentParser(description=description) CIME.utils.setup_standard_logging_options(parser) parser.add_argument("caseroot", nargs="?", default=os.getcwd(), help="Case directory to build") + parser.add_argument('--component', + help="Specify component's namelist to build.") + parser.add_argument('--test', action='store_true', + help="Run preview_namelist in test mode.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - return args.caseroot + return args ############################################################################### def _main_func(description): ############################################################################### - if ("--test" in sys.argv): + args = parse_command_line(sys.argv, description) + if args.test: test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - caseroot = parse_command_line(sys.argv, description) - expect(os.path.isfile(os.path.join(caseroot, "CaseStatus")), + expect(os.path.isfile(os.path.join(args.caseroot, "CaseStatus")), "case.setup must be run prior to running preview_namelists") - with Case(caseroot, read_only=False) as case: - create_namelists(case) + with Case(args.caseroot, read_only=False) as case: + create_namelists(case, component=args.component) if (__name__ == "__main__"): _main_func(__doc__) diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run new file mode 100755 index 000000000000..eca424d052a0 --- /dev/null +++ b/scripts/Tools/preview_run @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +""" +Script to query key CIME shell commands (mpirun and batch submission). + +To force a certain mpirun command, use: +./xmlchange MPI_RUN_COMMAND=$your_cmd + +Example: +./xmlchange MPI_RUN_COMMAND='mpiexec -np 16 --some-flag' + +To force a certain qsub command, use: +./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS=$your_flags + +Example: +./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS='--some-flag --other-flag' +""" + +from standard_script_setup import * + +from CIME.case import Case + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Run the tool \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory to build") + + args = parser.parse_args(args[1:]) + + return args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + if "--test" in sys.argv: + test_results = doctest.testmod(verbose=True) + sys.exit(1 if test_results.failed > 0 else 0) + + caseroot = parse_command_line(sys.argv, description) + + logging.disable(logging.CRITICAL) + + with Case(caseroot, read_only=False) as case: + print "BATCH SUBMIT:" + job = "case.test" if case.get_value("TEST") else "case.run" + job_id_to_cmd = case.submit_jobs(dry_run=True, job=job) + for job_id, cmd in job_id_to_cmd: + print " ", job_id, "->", case.get_resolved_value(cmd) + print + print "MPIRUN:", case.get_resolved_value(case.get_mpirun_cmd()) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/scripts/Tools/save_provenance b/scripts/Tools/save_provenance index 885c55a4eb6e..fa3ae37b869d 100755 --- a/scripts/Tools/save_provenance +++ b/scripts/Tools/save_provenance @@ -42,9 +42,7 @@ OR parser.add_argument("-l", "--lid", help="Force system to save provenance with this LID") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.mode, args.caseroot, args.lid diff --git a/scripts/Tools/simple_compare b/scripts/Tools/simple_compare index 47b465c14fa5..a72ebe398bc1 100755 --- a/scripts/Tools/simple_compare +++ b/scripts/Tools/simple_compare @@ -40,9 +40,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-c", "--case", action="store", dest="case", default=None, help="The case base id (..). Helps us normalize data.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) # Normalize case if (args.case is not None): diff --git a/scripts/Tools/testreporter.py b/scripts/Tools/testreporter.py new file mode 100755 index 000000000000..bc5188ab41c8 --- /dev/null +++ b/scripts/Tools/testreporter.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python + +""" + +Simple script to populate CESM test database with test results. + +""" + +from standard_script_setup import * + +from CIME.XML.env_build import EnvBuild +from CIME.XML.env_case import EnvCase +from CIME.XML.env_test import EnvTest +from CIME.XML.test_reporter import TestReporter +from CIME.utils import expect +from CIME.XML.generic_xml import GenericXML + + +import glob + + + +############################################################################### +def parse_command_line(args): +############################################################################### + parser = argparse.ArgumentParser() + + CIME.utils.setup_standard_logging_options(parser) + + # Parse command line options + +# parser = argparse.ArgumentParser(description='Arguements for testreporter') + parser.add_argument("--tagname", + help="Name of the tag being tested.") + parser.add_argument("--testid", + help="Test id, ie c2_0_a6g_ing,c2_0_b6g_gnu.") + parser.add_argument("--testroot", + help="Root directory for tests to populate the database.") + parser.add_argument("--testtype", + help="Type of test, prealpha or prebeta.") + parser.add_argument("--dryrun",action="store_true", + help="Do a dry run, database will not be populated.") + parser.add_argument("--dumpxml",action="store_true", + help="Dump XML test results to sceen.") + args = parser.parse_args() + CIME.utils.parse_args_and_handle_standard_logging_options(args) + + + return args.testroot, args.testid, args.tagname, args.testtype, args.dryrun, args.dumpxml + +############################################################################### +def get_testreporter_xml(testroot, testid, tagname, testtype): +############################################################################### + os.chdir(testroot) + + # + # Retrieve compiler name and mpi library + # + xml_file=glob.glob("*"+testid+"/env_build.xml") + expect(len(xml_file) > 0, "Tests not found. It's possible your testid, %s is wrong." %testid ) + envxml=(EnvBuild(".",infile=xml_file[0])) + compiler=envxml.get_value("COMPILER") + mpilib=envxml.get_value("MPILIB") + + # + # Retrieve machine name + # + xml_file=glob.glob("*"+testid+"/env_case.xml") + envxml=(EnvCase(".",infile=xml_file[0])) + machine=envxml.get_value("MACH") + + # + # Retrieve baseline tag to compare to + # + xml_file=glob.glob("*"+testid+"/env_test.xml") + envxml=(EnvTest(".",infile=xml_file[0])) + baseline = envxml.get_value("BASELINE_NAME_CMP") + + # + # Create XML header + # + + testxml=TestReporter() + testxml.setup_header(tagname,machine,compiler,mpilib,testroot,testtype,baseline) + + # + # Create lists on tests based on the testid in the testroot directory. + # + test_names=glob.glob("*"+testid) + # + # Loop over all tests and parse the test results + # + test_status={} + for test_name in test_names: + if not os.path.isfile(test_name+"/TestStatus"): + continue + test_status['COMMENT']="" + test_status['BASELINE']='----' + test_status['MEMCOMP']='----' + test_status['MEMLEAK']='----' + test_status['NLCOMP']='----' + test_status['STATUS']='----' + test_status['TPUTCOMP']='----' + # + # Check to see if TestStatus is present, if not then continue + # I might want to set the status to fail + # + try: + lines = [line.rstrip('\n') for line in open(test_name+"/TestStatus")] + except: + test_status['STATUS']="FAIL" + test_status['COMMENT']="TestStatus missing. " + continue + # + # Loop over each line of TestStatus, and check for different types of failures. + # + for line in lines: + if "NLCOMP" in line: + test_status['NLCOMP']=line[0:4] + if "MEMLEAK" in line: + test_status['MEMLEAK']=line[0:4] + if "MEMCOMP" in line: + test_status['MEMCOMP']=line[0:4] + if "BASELINE" in line: + test_status['BASELINE']=line[0:4] + if "TPUTCOMP" in line: + test_status['TPUTCOMP']=line[0:4] + if "INIT" in line: + test_status['INIT']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="SFAIL" + test_status['COMMENT']+="INIT fail! " + break + if "CREATE_NEWCASE" in line: + test_status['CREATE_NEWCASE']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="SFAIL" + test_status['COMMENT']+="CREATE_NEWCASE fail! " + break + if "XML" in line: + test_status['XML']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="SFAIL" + test_status['COMMENT']+="XML fail! " + break + if "SETUP" in line: + test_status['SETUP']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="SFAIL" + test_status['COMMENT']+="SETUP fail! " + break + if "SHAREDLIB_BUILD" in line: + test_status['SHAREDLIB_BUILD']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="CFAIL" + test_status['COMMENT']+="SHAREDLIB_BUILD fail! " + break + if "MODEL_BUILD" in line: + test_status['MODEL_BUILD']=line[0:4] + if line[0:4] == "FAIL": + test_status['STATUS']="CFAIL" + test_status['COMMENT']+="MODEL_BUILD fail! " + break + if "SUBMIT" in line: + test_status['STATUS']=line[0:4] + if line[0:4] == "FAIL": + test_status['COMMENT']+="SUBMIT fail! " + break + if "RUN" in line: + test_status['STATUS']=line[0:4] + if line[0:4] == "FAIL": + test_status['COMMENT']+="RUN fail! " + break + if "COMPARE_base_rest" in line: + test_status['STATUS']=line[0:4] + if line[0:4] == "FAIL": + test_status['COMMENT']+="Restart fail! " + break + if "COMPARE_base_hybrid" in line: + test_status['STATUS']=line[0:4] + if line[0:4] == "FAIL": + test_status['COMMENT']+="Hybrid fail! " + break + + # + # Do not include time comments. Just a preference to have cleaner comments in the test database + # + try: + if 'time=' not in line: + test_status['COMMENT']+=line.split(' ',3)[3]+' ' + except: + pass + + # + # Fill in the xml with the test results + # + testxml.add_result(test_name,test_status) + + return testxml + + +############################################################################## +def _main_func(): +############################################################################### + + testroot, testid, tagname, testtype, dryrun, dumpxml = parse_command_line(sys.argv) + + testxml = get_testreporter_xml(testroot, testid, tagname, testtype) + + # + # Dump xml to a file. + # + if dumpxml: + GenericXML.write(testxml,outfile="TestRecord.xml") + + + # + # Prompt for username and password, then post the XML string to the test database website + # + if not dryrun: + testxml.push2testdb() + +############################################################################### + +if __name__ == "__main__": + _main_func() + + diff --git a/scripts/Tools/update_acme_tests b/scripts/Tools/update_acme_tests index f5aeb4a0207c..588c51321604 100755 --- a/scripts/Tools/update_acme_tests +++ b/scripts/Tools/update_acme_tests @@ -48,9 +48,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-p", "--platform", help="Only add tests for a specific platform, format=machine,compiler. Useful for adding new platforms.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) expect(os.path.isfile(args.test_list_path), "'%s' is not a valid file" % args.test_list_path) diff --git a/scripts/Tools/wait_for_tests b/scripts/Tools/wait_for_tests index 3e1b88d343f5..7b4e698f3fa0 100755 --- a/scripts/Tools/wait_for_tests +++ b/scripts/Tools/wait_for_tests @@ -66,9 +66,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-g", "--cdash-build-group", default=CIME.wait_for_tests.CDASH_DEFAULT_BUILD_GROUP, help="The build group to be used to display results on the CDash dashboard.") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.paths, args.no_wait, args.check_throughput, args.check_memory, args.ignore_namelist_diffs, args.ignore_memleak, args.cdash_build_name, args.cdash_project, args.cdash_build_group diff --git a/scripts/Tools/xmlchange b/scripts/Tools/xmlchange index 57581e506f5c..f32bf556c0d8 100755 --- a/scripts/Tools/xmlchange +++ b/scripts/Tools/xmlchange @@ -24,7 +24,7 @@ logger = logging.getLogger("xmlchange") def parse_command_line(args, description): ############################################################################### parser = argparse.ArgumentParser( - usage="""\n%s [] [--verbose][--file file][--id id][--val value][--noecho][--append][--warn][--force] + usage="""\n%s [] [--verbose][--file file][--id id][--val value][--noecho][--append][--force] OR %s --help OR @@ -72,31 +72,30 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-append","--append", action="store_true", help="append to the existing value") - parser.add_argument("-warn","--warn", action="store_true", - help="not implemented") - parser.add_argument("-subgroup","--subgroup", help="apply to this subgroup only") parser.add_argument("-f","--force", action="store_true", help="ignore typing checks and store value") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) listofsettings = [] if( len(args.listofsettings )): delimiter = re.escape(args.delimiter) listofsettings = re.split(r'(? 0 else 0) # pylint: disable=unused-variable - caseroot, listofsettings, xmlfile, xmlid, xmlval, subgroup, append, noecho, warn, force , dry = parse_command_line(sys.argv, description) + caseroot, listofsettings, xmlfile, xmlid, xmlval, subgroup, append, noecho, force , dry = parse_command_line(sys.argv, description) - xmlchange(caseroot, listofsettings, xmlid, xmlval, subgroup, append, noecho, force , dry) + xmlchange(caseroot, listofsettings, xmlfile, xmlid, xmlval, subgroup, append, noecho, force, dry) if (__name__ == "__main__"): _main_func(__doc__) diff --git a/scripts/Tools/xmlconvertors/config_pes_converter.py b/scripts/Tools/xmlconvertors/config_pes_converter.py index a0d0746c2275..c453ee4c31b7 100755 --- a/scripts/Tools/xmlconvertors/config_pes_converter.py +++ b/scripts/Tools/xmlconvertors/config_pes_converter.py @@ -28,9 +28,7 @@ def parse_command_line(args): parser.add_argument("-cime2file", "--cime2file", help="location of config_grid.xml file in CIME2 repository") parser.add_argument("-cime5file", "--cime5file", help="location of config_grids.xml file in CIME5 repository") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.cime2file is None or args.cime5file is None: parser.print_help() @@ -187,8 +185,8 @@ def __init__(self, xmlfilename): tempxml = StringIO.StringIO(t3) super(PesTree, self).__init__(tempxml) tempxml.close() - - + + def populate(self): xmlnodes = self.root.findall('grid') nodeclass = Cime5PesNode diff --git a/scripts/Tools/xmlconvertors/grid_xml_converter.py b/scripts/Tools/xmlconvertors/grid_xml_converter.py index 28fb95f49b50..988727a3edcc 100755 --- a/scripts/Tools/xmlconvertors/grid_xml_converter.py +++ b/scripts/Tools/xmlconvertors/grid_xml_converter.py @@ -33,9 +33,7 @@ def parse_command_line(args): parser.add_argument("-cime2file", "--cime2file", help="location of config_grid.xml file in CIME2 repository") parser.add_argument("-cime5file", "--cime5file", help="location of config_grids.xml file in CIME5 repository") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.cime2file is None or args.cime5file is None: parser.print_help() diff --git a/scripts/Tools/xmlquery b/scripts/Tools/xmlquery index 0b3177004dc4..26b6b23d788b 100755 --- a/scripts/Tools/xmlquery +++ b/scripts/Tools/xmlquery @@ -63,7 +63,7 @@ epilog=textwrap.dedent(__doc__)) help="variable name in env_*.xml file ( value )") parser.add_argument("-file" , "--file", - help="deprecated option, do not use") + help="specify the file you want to query") parser.add_argument("-subgroup","--subgroup", help="apply to this subgroup only") @@ -105,9 +105,7 @@ epilog=textwrap.dedent(__doc__)) group.add_argument("--valid-values", default=False, action="store_true", help="Print the valid values associated with this variable if defined") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if (len(sys.argv) == 1) : parser.print_help() @@ -120,7 +118,7 @@ epilog=textwrap.dedent(__doc__)) return variables, args.subgroup, args.caseroot, args.listall, args.fileonly, \ args.value, args.no_resolve, args.raw, args.description, args.group, args.full, \ - args.type, args.valid_values, args.partial_match + args.type, args.valid_values, args.partial_match, args.file def get_value_as_string(case, var, attribute=None, resolved=False, subgroup=None): thistype = case.get_type_info(var) @@ -131,20 +129,30 @@ def get_value_as_string(case, var, attribute=None, resolved=False, subgroup=None def xmlquery(case, variables, subgroup=None, fileonly=False, resolved=True, raw=False, description=False, group=False, - full=False, dtype=False, valid_values=False): + full=False, dtype=False, valid_values=False, xmlfile=None): """ Return list of attributes and their values, print formatted """ results = {} comp_classes = case.get_values("COMP_CLASSES") + if xmlfile: + case.set_file(xmlfile) for var in variables: - if subgroup is not None: + if xmlfile: + groups = ['none'] + elif subgroup is not None: groups = [subgroup] else: groups = case.get_record_fields(var, "group") - expect(groups, " No results found for variable %s"%var) + if xmlfile: + value = case.get_value(var, resolved=resolved) + results['none'] = {} + results['none'][var] = {} + results['none'][var]['value'] = value + else: + expect(groups, " No results found for variable %s"%var) for group in groups: if not group in results: results[group] = {} @@ -164,7 +172,11 @@ def xmlquery(case, variables, subgroup=None, fileonly=False, else: value = get_value_as_string(case, var, resolved=resolved, subgroup=group) - expect(value is not None, " No results found for variable %s"%var) + if value is None: + if xmlfile: + expect(False, " No results found for variable %s in file %s"%(var, xmlfile)) + else: + expect(False, " No results found for variable %s"%var) results[group][var]['value'] = value @@ -189,7 +201,7 @@ def _main_func(): # Initialize command line parser and get command line options variables, subgroup, caseroot, listall, fileonly, \ value, no_resolve, raw, description, group, full, dtype, \ - valid_values, partial_match = parse_command_line(sys.argv) + valid_values, partial_match, xmlfile = parse_command_line(sys.argv) # Initialize case ; read in all xml files from caseroot with Case(caseroot) as case: @@ -226,7 +238,7 @@ def _main_func(): expect(variables, "No variables found") results = xmlquery(case, variables, subgroup, fileonly, resolved=not no_resolve, raw=raw, description=description, group=group, full=full, - dtype=dtype, valid_values=valid_values) + dtype=dtype, valid_values=valid_values, xmlfile=xmlfile) if full or description: wrapper=textwrap.TextWrapper() diff --git a/scripts/create_clone b/scripts/create_clone index 968848a7a425..74c7bedfee5d 100755 --- a/scripts/create_clone +++ b/scripts/create_clone @@ -40,9 +40,7 @@ def parse_command_line(args): help="Specify the root output directory" "default: setting in case, create_clone will fail if this directory is not writable") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.case is None: expect(False, diff --git a/scripts/create_newcase b/scripts/create_newcase index f1877cd76e5a..627c0b60f42f 100755 --- a/scripts/create_newcase +++ b/scripts/create_newcase @@ -28,7 +28,7 @@ OR parser.add_argument("--case", "-case", required=True, help="(required) Specify the case name. " "If not a full pathname, then the case is created " - "under then current working directory ") + "under then current working directory ") parser.add_argument("--compset", "-compset", required=True, help="(required) Specify a compset. " @@ -76,7 +76,7 @@ OR parser.add_argument("--pesfile", help="Only used and required for --user-compset argument." "Full pathname of the pes specification file" - "This argument is required if --user-compset is True") + "This argument is required if --user-compset is True") parser.add_argument("--user-grid", action="store_true", help="If set, then the -grid argument is treated as a user specified grid." @@ -88,7 +88,7 @@ OR help="Full pathname of config grid file to use" "This should be a copy of config/config_grids.xml" "with the new user grid changes added to it" - "This argument is required if --user-grid is True") + "This argument is required if --user-grid is True") parser.add_argument("--srcroot", default=os.path.dirname(cimeroot), help="Alternative path for source root directory. By default this is set to" @@ -97,6 +97,9 @@ OR parser.add_argument("--output-root", help="Alternative path for the directory where case output is written") + parser.add_argument("--script-root", dest="script_root", default=None, + help="Alternative path for the directory where the cime scripts are written") + if model == "cesm": parser.add_argument("--run-unsupported", action="store_true", help="Force the creation of a case not tested or supported by CESM developers") @@ -118,9 +121,10 @@ OR help="Pre-answer what to do pre-existing bld/exe dirs. Valid answers are (a)bort (r)eplace or (u)se existing." "This can be useful if you need to run create_newcase non-iteractively.") - args = parser.parse_args() + parser.add_argument("-i", "--input-dir", + help="Use a non-default location for input files") - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if args.srcroot is not None: expect(os.path.isdir(args.srcroot), @@ -150,24 +154,32 @@ OR expect(CIME.utils.check_name(args.case, fullpath=True), "Illegal case name argument provided") + if args.input_dir is not None: + args.input_dir = os.path.abspath(args.input_dir) + return args.case, args.compset, args.res, args.machine, args.compiler,\ args.mpilib, args.project, args.pecount, \ args.user_mods_dir, args.user_compset, args.pesfile, \ args.user_grid, args.gridfile, args.srcroot, args.test, args.ninst, \ - args.walltime, args.queue, args.output_root, run_unsupported, args.answer + args.walltime, args.queue, args.output_root, args.script_root, \ + run_unsupported, args.answer, args.input_dir ############################################################################### def _main_func(description): ############################################################################### cimeroot = os.path.abspath(CIME.utils.get_cime_root()) - caseroot, compset, grid, machine, compiler, \ + casename, compset, grid, machine, compiler, \ mpilib, project, pecount, \ user_mods_dir, user_compset, pesfile, \ user_grid, gridfile, srcroot, test, ninst, walltime, queue, \ - output_root, run_unsupported, answer = parse_command_line(sys.argv, cimeroot, description) + output_root, script_root, run_unsupported, \ + answer, input_dir = parse_command_line(sys.argv, cimeroot, description) - caseroot = os.path.abspath(caseroot) + if script_root is None: + caseroot = os.path.abspath(casename) + else: + caseroot = os.path.abspath(script_root) # create_test creates the caseroot before calling create_newcase # otherwise throw an error if this directory exists @@ -176,7 +188,7 @@ def _main_func(description): with Case(caseroot, read_only=False) as case: # Set values for env_case.xml - case.set_lookup_value("CASE", os.path.basename(caseroot)) + case.set_lookup_value("CASE", os.path.basename(casename)) case.set_lookup_value("CASEROOT", caseroot) case.set_lookup_value("SRCROOT", srcroot) @@ -186,7 +198,8 @@ def _main_func(description): user_compset=user_compset, pesfile=pesfile, user_grid=user_grid, gridfile=gridfile, ninst=ninst, test=test, walltime=walltime, queue=queue, output_root=output_root, - run_unsupported=run_unsupported, answer=answer) + run_unsupported=run_unsupported, answer=answer, + input_dir=input_dir) case.create_caseroot() @@ -195,7 +208,7 @@ def _main_func(description): if user_mods_dir is not None: if os.path.isdir(user_mods_dir): user_mods_dir = os.path.abspath(user_mods_dir) - case.apply_user_mods(user_mods_dir) + case.apply_user_mods(user_mods_dir) # Lock env_case.xml lock_file("env_case.xml", caseroot) diff --git a/scripts/create_test b/scripts/create_test index f1dcede9b6e2..356545789548 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -20,7 +20,6 @@ import argparse, math, glob logger = logging.getLogger(__name__) - ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -226,9 +225,7 @@ OR parser.add_argument("-i", "--input-dir", help="Use a non-default location for input files") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) # generate and compare flags may not point to the same directory if model == "cesm": @@ -412,7 +409,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost submit_cmd = env_batch.get_value("batch_submit", subgroup=None) submit_args = env_batch.get_submit_args(case, "case.run") - tasks_per_node = int(mach.get_value("PES_PER_NODE")) + tasks_per_node = mach.get_value("PES_PER_NODE") num_nodes = int(math.ceil(float(proc_pool) / tasks_per_node)) if wall_time is None: wall_time = compute_total_time(job_cost_map, proc_pool) @@ -420,8 +417,8 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost else: wall_time_bab = wall_time - queue = env_batch.select_best_queue(proc_pool) - wall_time_max_bab = env_batch.get_max_walltime(queue) + queue = env_batch.select_best_queue(proc_pool, wall_time_bab) + wall_time_max_bab = env_batch.get_queue_specs(queue)[3] if wall_time_max_bab is not None: wall_time_max = convert_to_seconds(wall_time_max_bab) if wall_time_max < wall_time: diff --git a/tools/unit_testing/Examples/circle_area/src/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/circle_area/src/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/circle_area/src/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/circle_area/src/CMakeLists.txt diff --git a/tools/unit_testing/Examples/circle_area/src/circle.F90 b/scripts/fortran_unit_testing/Examples/circle_area/src/circle.F90 similarity index 100% rename from tools/unit_testing/Examples/circle_area/src/circle.F90 rename to scripts/fortran_unit_testing/Examples/circle_area/src/circle.F90 diff --git a/tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/circle_area/tests/CMakeLists.txt similarity index 89% rename from tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/circle_area/tests/CMakeLists.txt index f7bcbf92b2fc..06b4b9ada401 100644 --- a/tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt +++ b/scripts/fortran_unit_testing/Examples/circle_area/tests/CMakeLists.txt @@ -1,7 +1,9 @@ cmake_minimum_required(VERSION 2.8) -project(circle_area_tests Fortran C) list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) +include(CIME_initial_setup) + +project(circle_area_tests Fortran C) include(CIME_utils) diff --git a/tools/unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt diff --git a/tools/unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 b/scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 similarity index 100% rename from tools/unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 rename to scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 diff --git a/tools/unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt diff --git a/tools/unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf b/scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf similarity index 100% rename from tools/unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf rename to scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf diff --git a/tools/unit_testing/Examples/interpolate_1d/src/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/interpolate_1d/src/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/src/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/interpolate_1d/src/CMakeLists.txt diff --git a/tools/unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 b/scripts/fortran_unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 rename to scripts/fortran_unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt similarity index 89% rename from tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt index 378330073d27..e1747bb812b7 100644 --- a/tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt +++ b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt @@ -1,7 +1,10 @@ cmake_minimum_required(VERSION 2.8) -project(interpolate_1d_tests Fortran C) list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) +include(CIME_initial_setup) + +project(interpolate_1d_tests Fortran C) + include(CIME_utils) add_subdirectory(../src interpolate_1d) diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 rename to scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt rename to scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf b/scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf similarity index 100% rename from tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf rename to scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf diff --git a/tools/unit_testing/Examples/test_list.xml b/scripts/fortran_unit_testing/Examples/test_list.xml similarity index 100% rename from tools/unit_testing/Examples/test_list.xml rename to scripts/fortran_unit_testing/Examples/test_list.xml diff --git a/tools/unit_testing/README b/scripts/fortran_unit_testing/README similarity index 84% rename from tools/unit_testing/README rename to scripts/fortran_unit_testing/README index 4b1435ff9043..8bd194ce57d6 100644 --- a/tools/unit_testing/README +++ b/scripts/fortran_unit_testing/README @@ -1,20 +1,10 @@ -*- mode:org -*- #+startup: showall -Quick guide to the CESM unit testing framework +Quick guide to the CIME unit testing framework * Requirements - - Machines :: Basic compiler information is retrieved from - config_compilers.xml in Machines. Even if running on an - unsupported machine, it is necessary to have this file to - get some basic information about the compiler. - - As of this writing, it is possible to run with Machines, - this directory, the CMake module directory, and the - directory that specifies tests. A full CESM checkout is - more convenient but not needed. - - Python 2.6 or later :: Required for scripts. Python 3.2 is also known to work. @@ -40,7 +30,7 @@ Quick guide to the CESM unit testing framework ** Quick start In the simplest case, you may want to run tests from a single directory. - In that case, there are just three pieces of information that you usually + In that case, there are just two pieces of information that you usually need to specify: 1. The location of the directory where you want to build and run the @@ -48,23 +38,11 @@ Quick guide to the CESM unit testing framework 2. The location of the directory specifying the test via a CMakeLists.txt file ("TEST_SPEC_DIR"). - - 3. The compiler vendor ("COMPILER"). + (Optional: Defaults to the current directory) The following command can run the tests: - run_tests.py --build-dir=BUILD_DIR --test-spec-dir=TEST_SPEC_DIR \ - --compiler=COMPILER - - The current compiler options are: - - - GNU - - IBM - - Intel - - NAG - - PGI - - COMPILER defaults to GNU. + run_tests.py --build-dir=BUILD_DIR --test-spec-dir=TEST_SPEC_DIR If you change a source file, you can run the same command again, and run_tests.py will do the bare minimum of work necessary to re-run the @@ -79,18 +57,9 @@ Quick guide to the CESM unit testing framework Some particularly useful options are as follows: - + --build-type :: Setting this option to "CESM" will build the tests - with CESM compiler options (or the most similar - pFUnit-compatible flags). Setting this option to - "CESM_DEBUG" will build with CESM's DEBUG mode - options. - + --clean :: Cleans the directory and re-runs both CMake and make when running the tests. - + -R :: This option allows you to specify a regular expression; only - tests with names matching that expression will run. - + -v :: Verbose output, including the commands used to compile, any compiler warnings, and CTest output from tests that pass. To get CTest output from tests that fail, set the environment variable @@ -103,34 +72,41 @@ Quick guide to the CESM unit testing framework * Defining new tests using CMake - This README focuses on integration between the python scripts here, the - CMake modules, and CESM Machines. Further information, such as detailed - APIs, can be found in the documentation for pFUnit and the CMake modules - themselves. + This README focuses on integration between the python scripts here, + the CMake modules, and machine information. Further information, such + as detailed APIs, can be found in the documentation for pFUnit and the + CMake modules themselves. If you are new to this system, it is *highly* recommended that you look through the examples first. -** CIME_utils - run_tests.py interface +** CIME_initial_setup and CIME_utils - run_tests.py interface - The following CMake snippet will include the CESM utilities module. The + The following CMake snippet will include the CIME utilities module. The variable ${CIME_CMAKE_MODULE_DIRECTORY} is defined by run_tests.py, or by hand if you choose not to use run_tests.py and instead invoke cmake directly. list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) + include(CIME_initial_setup) + project(cime_tests Fortran C) include(CIME_utils) + The project name does not need to be 'cime_tests'. The key point of + the above snippet is that you need to include CIME_initial_setup + before the project line, and CIME_utils after the project line. + CIME_utils processes a few options set by run_tests.py (e.g. "USE_COLOR"), and includes all of the following modules as well. Projects that do development without run_tests.py may choose to include only the modules below that they need. -** Compilers - CESM compiler options +** Compilers - CIME compiler options This module is also part of the run_tests.py interface; the primary - purpose is to read in flags generated from CESM's config_compilers.xml - file. However, it's also a catch-all for compiler-specific information. + purpose is to read in flags generated from the config_compilers.xml + file. However, it's also a catch-all for compiler-specific + information. The details of this module shouldn't be important to most users, most of the time, but it does provide one utility function that you may want to diff --git a/tools/unit_testing/python/.gitignore b/scripts/fortran_unit_testing/python/.gitignore similarity index 100% rename from tools/unit_testing/python/.gitignore rename to scripts/fortran_unit_testing/python/.gitignore diff --git a/tools/unit_testing/python/printer.py b/scripts/fortran_unit_testing/python/printer.py similarity index 100% rename from tools/unit_testing/python/printer.py rename to scripts/fortran_unit_testing/python/printer.py diff --git a/tools/unit_testing/python/test_xml_test_list.py b/scripts/fortran_unit_testing/python/test_xml_test_list.py similarity index 97% rename from tools/unit_testing/python/test_xml_test_list.py rename to scripts/fortran_unit_testing/python/test_xml_test_list.py index 2aecd4d83987..565f12d89f6a 100755 --- a/tools/unit_testing/python/test_xml_test_list.py +++ b/scripts/fortran_unit_testing/python/test_xml_test_list.py @@ -58,7 +58,7 @@ class TestSuitesFromXML(unittest.TestCase): """Tests for the suites_from_xml function.""" def check_spec_list(self, xml_str, names, directories, - known_paths={}, labels=None): + known_paths=None, labels=None): """Check that a spec list matches input names and directories. This is used by the following tests to do the dirty work of making @@ -129,7 +129,6 @@ def test_multiple_suites(self): def test_path_relative_to_known(self): """suites_from_xml handles a relative_to directory attribute.""" - from os.path import abspath xml_str = """ @@ -145,7 +144,6 @@ def test_path_relative_to_known(self): def test_path_with_whitespace(self): """suites_from_xml handles a directory with whitespace added.""" - from os.path import abspath xml_str = """ @@ -160,7 +158,6 @@ def test_path_with_whitespace(self): def test_path_with_label(self): """suites_from_xml handles a directory with a label correctly.""" - from os.path import abspath xml_str = """ diff --git a/tools/unit_testing/python/xml_test_list.py b/scripts/fortran_unit_testing/python/xml_test_list.py similarity index 96% rename from tools/unit_testing/python/xml_test_list.py rename to scripts/fortran_unit_testing/python/xml_test_list.py index 9e7007f93e87..ec0cf68796e4 100644 --- a/tools/unit_testing/python/xml_test_list.py +++ b/scripts/fortran_unit_testing/python/xml_test_list.py @@ -58,7 +58,7 @@ def __iter__(self): """ return ( (l, d) for l, d in zip(self.labels, self.directories) ) -def suites_from_xml(xml_tree, known_paths={}): +def suites_from_xml(xml_tree, known_paths=None): """Generate test suite descriptions from XML. Returns a TestSuiteSpec for each suite description in the XML input. @@ -77,6 +77,9 @@ def suites_from_xml(xml_tree, known_paths={}): the known_paths dict. """ + if known_paths is None: + known_paths = {} + elements = xml_tree.findall("suite") for elem in elements: diff --git a/tools/unit_testing/run_tests.py b/scripts/fortran_unit_testing/run_tests.py similarity index 71% rename from tools/unit_testing/run_tests.py rename to scripts/fortran_unit_testing/run_tests.py index e7a152fe2009..9ce19b2d4535 100755 --- a/tools/unit_testing/run_tests.py +++ b/scripts/fortran_unit_testing/run_tests.py @@ -4,21 +4,22 @@ _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) sys.path.append(os.path.join(_CIMEROOT, "scripts", "utils", "python")) -sys.path.append(os.path.join(_CIMEROOT, "tools", "unit_testing", "python")) +sys.path.append(os.path.join(_CIMEROOT, "scripts", "fortran_unit_testing", "python")) from standard_script_setup import * from CIME.BuildTools.configure import configure -from CIME.utils import run_cmd_no_fail, stringify_bool +from CIME.utils import run_cmd_no_fail, stringify_bool, expect from CIME.XML.machines import Machines +from CIME.XML.compilers import Compilers from CIME.XML.env_mach_specific import EnvMachSpecific from xml_test_list import TestSuiteSpec, suites_from_xml import subprocess +import socket #================================================= # Standard library modules. #================================================= from printer import Printer from shutil import rmtree -from distutils.spawn import find_executable # This violates CIME policy - move to CIME/XML directory from xml.etree.ElementTree import ElementTree @@ -81,21 +82,24 @@ def parse_command_line(args): help="""Number of processes to use for build.""" ) + parser.add_argument("--use-mpi", action="store_true", + help="""If specified, run unit tests with an mpi-enabled version + of pFUnit, via mpirun. (Default is to use a serial build without + mpirun.) This requires a pFUnit build with MPI support.""") + parser.add_argument("--mpilib", - help="""MPI Library to use in build. - If not specified, use the default for this machine/compiler. - Must match an MPILIB option in config_compilers.xml. - e.g., for yellowstone, can use 'mpich2'.""" + help="""MPI Library to use in build. + If not specified, use the default for this machine/compiler. + Must match an MPILIB option in config_compilers.xml. + e.g., for yellowstone, can use 'mpich2'. + Only relevant if --use-mpi is specified.""" ) - parser.add_argument("--no-mpirun", action="store_true", - help="""If specified, then run executables without an mpirun command""") - parser.add_argument("--mpirun-command", - help="""Command to use to run an MPI executable. - If not specified, uses the default for this machine. - Ignored if --no-mpirun is specified.""" - ) + help="""Command to use to run an MPI executable. + If not specified, uses the default for this machine. + Only relevant if --use-mpi is specified.""" + ) parser.add_argument( "--test-spec-dir", default=".", help="""Location where tests are specified. @@ -114,15 +118,17 @@ def parse_command_line(args): override the command provided by Machines.""" ) parser.add_argument( - "--no-openmp", action="store_true", - help="""Default is to include OPENMP support for tests; this option excludes openmp""" + "--use-openmp", action="store_true", + help="""If specified, include OpenMP support for tests. + (Default is to run without OpenMP support.) This requires a pFUnit build with + OpenMP support.""" ) parser.add_argument( "--xml-test-list", help="""Path to an XML file listing directories to run tests from.""" ) - args = parser.parse_args() - CIME.utils.handle_standard_logging_options(args) + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) output = Printer(color=args.color) if args.xml_test_list is None and args.test_spec_dir is None: @@ -136,16 +142,19 @@ def parse_command_line(args): return output, args.build_dir, args.build_optimized, args.clean,\ args.cmake_args, args.compiler, args.enable_genf90, args.machine, args.machines_dir,\ - args.make_j, args.mpilib, args.no_mpirun, args.mpirun_command, args.test_spec_dir, args.ctest_args,\ - args.no_openmp, args.xml_test_list, args.verbose + args.make_j, args.use_mpi, args.mpilib, args.mpirun_command, args.test_spec_dir, args.ctest_args,\ + args.use_openmp, args.xml_test_list, args.verbose -def cmake_stage(name, test_spec_dir, build_optimized, mpirun_command, output, cmake_args=None, clean=False, verbose=False, enable_genf90=True, color=True): +def cmake_stage(name, test_spec_dir, build_optimized, use_mpiserial, mpirun_command, output, + cmake_args=None, clean=False, verbose=False, enable_genf90=True, color=True): """Run cmake in the current working directory. Arguments: name - Name for output messages. test_spec_dir - Test specification directory to run CMake on. + use_mpiserial (logical) - If True, we'll tell CMake to include mpi-serial for tests + that need it build_optimized (logical) - If True, we'll build in optimized rather than debug mode """ # Clear CMake cache. @@ -174,6 +183,8 @@ def cmake_stage(name, test_spec_dir, build_optimized, mpirun_command, output, cm "-DCMAKE_BUILD_TYPE="+build_type, "-DPFUNIT_MPIRUN='"+mpirun_command+"'", ] + if use_mpiserial: + cmake_command.append("-DUSE_MPI_SERIAL=ON") if verbose: cmake_command.append("-Wdev") @@ -211,6 +222,26 @@ def make_stage(name, output, make_j, clean=False, verbose=True): run_cmd_no_fail(" ".join(make_command), arg_stdout=None, arg_stderr=subprocess.STDOUT) +def find_pfunit(compilerobj, mpilib, use_openmp): + """Find the pfunit installation we'll be using, and print its path + + Aborts if necessary information cannot be found. + + Args: + - compilerobj: Object of type Compilers + - mpilib: String giving the mpi library we're using + - use_openmp: Boolean + """ + attrs = {"MPILIB": mpilib, + "compile_threaded": "true" if use_openmp else "false" + } + + pfunit_path = compilerobj.get_optional_compiler_node("PFUNIT_PATH", attributes=attrs) + expect(pfunit_path is not None, + """PFUNIT_PATH not found for this machine and compiler, with MPILIB=%s and compile_threaded=%s. +You must specify PFUNIT_PATH in config_compilers.xml, with attributes MPILIB and compile_threaded."""%(mpilib, attrs['compile_threaded'])) + logger.info("Using PFUNIT_PATH: %s"%pfunit_path.text) + #================================================= # Iterate over input suite specs, building the tests. #================================================= @@ -219,12 +250,10 @@ def make_stage(name, output, make_j, clean=False, verbose=True): def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ - make_j, mpilib, no_mpirun, mpirun_command, test_spec_dir, ctest_args,\ - no_openmp, xml_test_list, verbose \ + make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ + use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) - use_openmp = not no_openmp - #================================================= # Find directory and file paths. #================================================= @@ -252,7 +281,7 @@ def _main(): else: machobj = Machines(machine=machine) -# Create build directory if necessary. + # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): @@ -261,15 +290,23 @@ def _main(): # Switch to the build directory. os.chdir(build_dir) -#================================================= -# Functions to perform various stages of build. -#================================================= - if mpilib is None: + #================================================= + # Functions to perform various stages of build. + #================================================= + + if not use_mpi: + mpilib = "mpi-serial" + elif mpilib is None: mpilib = machobj.get_default_MPIlib() - logger.warn("Using mpilib: %s"%mpilib) + logger.info("Using mpilib: %s"%mpilib) + if compiler is None: compiler = machobj.get_default_compiler() - logger.warn("Compiler is %s"%compiler) + logger.info("Compiler is %s"%compiler) + + compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) + + find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") @@ -277,10 +314,28 @@ def _main(): # Create the environment, and the Macros.cmake file # # - configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_) - machspecific = EnvMachSpecific(build_dir) + configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, + unit_testing=True) + machspecific = EnvMachSpecific(build_dir, unit_testing=True) - if no_mpirun: + machspecific.load_env(compiler, debug, mpilib) + os.environ["OS"] = os_ + os.environ["COMPILER"] = compiler + os.environ["DEBUG"] = stringify_bool(debug) + os.environ["MPILIB"] = mpilib + if use_openmp: + os.environ["compile_threaded"] = "true" + else: + os.environ["compile_threaded"] = "false" + + os.environ["UNIT_TEST_HOST"] = socket.gethostname() + if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: + # The CMake Netcdf find utility that we use (from pio2) seems to key off + # of the environment variable NETCDF, but not NETCDF_PATH + logger.info("Setting NETCDF environment variable: %s"%os.environ["NETCDF_PATH"]) + os.environ["NETCDF"] = os.environ["NETCDF_PATH"] + + if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { @@ -292,17 +347,8 @@ def _main(): # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True) - logger.warn("mpirun command is '%s'"%mpirun_command) - - machspecific.load_env(compiler, debug, mpilib) - os.environ["OS"] = os_ - os.environ["COMPILER"] = compiler - os.environ["DEBUG"] = stringify_bool(debug) - os.environ["MPILIB"] = mpilib - if use_openmp: - os.environ["compile_threaded"] = "true" - os.environ["CC"] = find_executable("mpicc") - os.environ["FC"] = find_executable("mpif90") + mpirun_command = machspecific.get_resolved_value(mpirun_command) + logger.info("mpirun command is '%s'"%mpirun_command) #================================================= # Run tests. @@ -328,7 +374,8 @@ def _main(): if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir,"Macros.cmake"), "Macros.cmake") - cmake_stage(name, directory, build_optimized, mpirun_command, output, verbose=verbose, + use_mpiserial = not use_mpi + cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) diff --git a/scripts/lib/CIME/BuildTools/cmakemacroswriter.py b/scripts/lib/CIME/BuildTools/cmakemacroswriter.py index 22ddac75eec9..a1789d983876 100644 --- a/scripts/lib/CIME/BuildTools/cmakemacroswriter.py +++ b/scripts/lib/CIME/BuildTools/cmakemacroswriter.py @@ -89,7 +89,8 @@ def set_variable(self, name, value): >>> s.getvalue() u'set(foo "bar")\\n' """ - self.write_line("set(" + name + ' "' + value + '")') + value_transformed = self._transform_value(value) + self.write_line("set(" + name + ' "' + value_transformed + '")') def start_ifeq(self, left, right): """Write out a statement to start a conditional block. @@ -117,3 +118,22 @@ def end_ifeq(self): """ self.indent_left() self.write_line("endif()") + + def _transform_value(self, value): + """Some elements need their values transformed in some way for CMake to handle them properly. + This method does those transformations. + + Args: + - value (str): value of element + + Returns transformed value + """ + + # Not all variables need leading & trailing whitespace removed, but some + # do. In particular, compiler variables (MPICC, MPICXX, MPIFC, SCC, + # SCXX, SFC) are only handled correctly if leading & trailing whitespace + # is removed. It doesn't seem to hurt to remove whitespace from all + # variables. + value_transformed = value.strip() + + return value_transformed diff --git a/scripts/lib/CIME/BuildTools/configure.py b/scripts/lib/CIME/BuildTools/configure.py index b610b087aa3c..26281117df62 100644 --- a/scripts/lib/CIME/BuildTools/configure.py +++ b/scripts/lib/CIME/BuildTools/configure.py @@ -23,7 +23,8 @@ logger = logging.getLogger(__name__) -def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, sysos): +def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, + sysos, unit_testing=False): """Add Macros, Depends, and env_mach_specific files to a directory. Arguments: @@ -34,6 +35,8 @@ def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, sysos compiler - String containing the compiler vendor to configure for. mpilib - String containing the MPI implementation to configure for. debug - Boolean specifying whether debugging options are enabled. + unit_testing - Boolean specifying whether we're running unit tests (as + opposed to a system run) """ # Macros generation. suffixes = {'Makefile': 'make', 'CMake': 'cmake'} @@ -44,7 +47,7 @@ def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, sysos _copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler) _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, - debug, sysos) + debug, sysos, unit_testing) def _copy_depends_files(machine_name, machines_dir, output_dir, compiler): """ @@ -64,7 +67,8 @@ def _copy_depends_files(machine_name, machines_dir, output_dir, compiler): shutil.copyfile(dfile, outputdfile) -def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos): +def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, + sysos, unit_testing): """ env_mach_specific generation. """ @@ -72,7 +76,7 @@ def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sy if os.path.exists(ems_path): logger.warn("%s already exists, delete to replace"%ems_path) return - ems_file = EnvMachSpecific(output_dir) + ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing) ems_file.populate(machobj) ems_file.write() for shell in ('sh', 'csh'): diff --git a/scripts/lib/CIME/SystemTests/erp.py b/scripts/lib/CIME/SystemTests/erp.py index 9bade606e290..5e0dc748b453 100644 --- a/scripts/lib/CIME/SystemTests/erp.py +++ b/scripts/lib/CIME/SystemTests/erp.py @@ -31,6 +31,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): and tasks. This test will fail for components (e.g. pop) that do not reproduce exactly with different numbers of mpi tasks. """ + self._case.set_value("BUILD_THREADED",True) if sharedlib_only: return self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) @@ -56,7 +57,6 @@ def build_phase(self, sharedlib_only=False, model_only=False): # The reason we currently need two executables that CESM-CICE has a compile time decomposition # For cases where ERP works, changing this decomposition will not affect answers, but it will # affect the executable that is used - self._case.set_value("SMP_BUILD","0") for bld in range(1,3): logging.warn("Starting bld %s"%bld) @@ -79,7 +79,6 @@ def build_phase(self, sharedlib_only=False, model_only=False): case_setup(self._case, test_mode=True, reset=True) # Now rebuild the system, given updated information in env_build.xml - self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) shutil.move("%s/%s.exe"%(exeroot,cime_model), "%s/%s.ERP%s.exe"%(exeroot,cime_model,bld)) diff --git a/scripts/lib/CIME/SystemTests/nck.py b/scripts/lib/CIME/SystemTests/nck.py index be474cbb4af4..e8c18caff179 100644 --- a/scripts/lib/CIME/SystemTests/nck.py +++ b/scripts/lib/CIME/SystemTests/nck.py @@ -46,7 +46,11 @@ def _case_one_setup(self): def _case_two_setup(self): for comp in self._comp_classes: - self._case.set_value("NINST_%s"%comp, 2) + if (comp == "ESP"): + self._case.set_value("NINST_%s"%comp, 1) + else: + self._case.set_value("NINST_%s"%comp, 2) + ntasks = self._case.get_value("NTASKS_%s"%comp) self._case.set_value("NTASKS_%s"%comp, ntasks*2) diff --git a/scripts/lib/CIME/SystemTests/nodefail.py b/scripts/lib/CIME/SystemTests/nodefail.py index 2f0436a870e6..9101a5579970 100644 --- a/scripts/lib/CIME/SystemTests/nodefail.py +++ b/scripts/lib/CIME/SystemTests/nodefail.py @@ -56,8 +56,16 @@ def _restart_fake_phase(self): env_mach_specific.set_value("run_exe", fake_exe_file) self._case.flush(flushall=True) + # This flag is needed by mpt to run a script under mpiexec + mpilib = self._case.get_value("MPILIB") + if mpilib == "mpt": + os.environ["MPI_SHEPHERD"] = "true" + self.run_indv(suffix=None) + if mpilib == "mpt": + del os.environ["MPI_SHEPHERD"] + env_mach_specific = self._case.get_env("mach_specific") env_mach_specific.set_value("run_exe", prev_run_exe) self._case.flush(flushall=True) diff --git a/scripts/lib/CIME/SystemTests/pre.py b/scripts/lib/CIME/SystemTests/pre.py index 22bb3f3bb207..5a9e8644f650 100644 --- a/scripts/lib/CIME/SystemTests/pre.py +++ b/scripts/lib/CIME/SystemTests/pre.py @@ -2,6 +2,7 @@ Implementation of the CIME pause/resume test: Tests having driver 'pause' (write cpl restart file) and 'resume' (read cpl restart file) without changing restart file. Compare to non-pause/resume run. +Test can also be run with other component combinations. """ @@ -19,8 +20,9 @@ class PRE(SystemTestsCompareTwo): ############################################################################### """ Implementation of the CIME pause/resume test: Tests having driver - 'pause' (write cpl restart file) and 'resume' (read cpl restart file) - without changing restart file. Compare to non-pause/resume run. + 'pause' (write cpl and/or other restart file(s)) and 'resume' + (read cpl and/or other restart file(s)) without changing restart + file. Compare to non-pause/resume run. """ ########################################################################### @@ -31,26 +33,35 @@ def __init__(self, case): run_two_suffix='pr', run_one_description='no pause/resume', run_two_description='pause/resume') + self._stopopt = '' + self._stopn = 0 ########################################################################### def _case_one_setup(self): ########################################################################### case_setup(self._case, test_mode=True, reset=True) + self._stopopt = self._case.get_value("STOP_OPTION") + self._stopn = self._case.get_value("STOP_N") ########################################################################### def _case_two_setup(self): ########################################################################### # Set up a pause/resume run - stopopt = self._case.get_value("STOP_OPTION") - stopn = self._case.get_value("STOP_N") - expect((stopn % 5) == 0, "ERROR: PRE test requires that STOP_N be divisible by five") - pausen = stopn / 5 - expect(pausen > 0, "ERROR: pause_n (%d) must be > 0, stop_n is %d"%(pausen, stopn)) - self._case.set_value("PAUSE_OPTION", stopopt) + self._case.set_value("STOP_OPTION", self._stopopt) + self._case.set_value("STOP_N", self._stopn) + self._case.set_value("ESP_RUN_ON_PAUSE", "TRUE") + if self._stopn > 3: + pausen = 2 + else: + pausen = 1 + # End if + + self._case.set_value("PAUSE_OPTION", self._stopopt) self._case.set_value("PAUSE_N", pausen) + comps = [ x.lower() for x in self._case.get_values("COMP_CLASSES") ] pcl = self._case.get_value("PAUSE_COMPONENT_LIST") - expect(pcl == "cpl", - "ERROR: PRE test expected PAUSE_COMPONENT_LIST = 'cpl', found '%s'"%pcl) + expect(pcl == "all" or set(pcl.split(':')).issubset(comps), + "PRE ERROR: Invalid PAUSE_COMPONENT_LIST, '%s'"%pcl) self._case.flush() @@ -65,37 +76,35 @@ def run_phase(self): self._activate_case2() rundir2 = self._case.get_value("RUNDIR") should_match = (self._case.get_value("DESP_MODE") == "NOCHANGE") + compare_ok = True pause_comps = self._case.get_value("PAUSE_COMPONENT_LIST") expect((pause_comps != 'none'), "Pause/Resume (PRE) test has no pause components") if pause_comps == 'all': pause_comps = self._case.get_values("COMP_CLASSES") else: pause_comps = pause_comps.split(':') - # End if + for comp in pause_comps: - restart_files_1 = glob.glob(os.path.join(rundir1, '*.%s.r.*'%comp)) - if len(restart_files_1) != 1: - logger.error("Wrong number of case1 %s restart files, %d", comp, len(restart_files_1)) - # End if - restart_files_2 = glob.glob(os.path.join(rundir2, '*.%s.r.*'%comp)) - if len(restart_files_2) != 5: - logger.error("Wrong number of case2 %s restart files, %d", comp, len(restart_files_2)) - # End if + comp_name = self._case.get_value('COMP_%s'%comp.upper()) + rname = '*.%s.r.*'%comp_name + restart_files_1 = glob.glob(os.path.join(rundir1, rname)) + expect((len(restart_files_1) > 0), "No case1 restart files for %s"%comp) + restart_files_2 = glob.glob(os.path.join(rundir2, rname)) + expect((len(restart_files_2) > len(restart_files_1)), + "No pause (restart) files found in case2 for %s"%comp) # Do cprnc of restart files. - rfile1 = restart_files_1[0] + rfile1 = restart_files_1[len(restart_files_1) - 1] # rfile2 has to match rfile1 (same time string) parts = os.path.basename(rfile1).split(".") glob_str = "*.%s"%".".join(parts[len(parts)-4:]) restart_files_2 = glob.glob(os.path.join(rundir2, glob_str)) - if len(restart_files_2) < 1: - logger.error("Missing case2 restart file, %s", glob_str) - # End if - if len(restart_files_2) > 1: - logger.error("Multiple case2 restart files, %s", glob_str) - # End if + expect((len(restart_files_2) == 1), + "Missing case2 restart file, %s", glob_str) rfile2 = restart_files_2[0] - ok, out = cprnc(comp, rfile1, rfile2, self._case, rundir2) - logger.warning("CPRNC result: %s, file = %s"%(ok, out)) - expect((should_match == ok), - "%s restart files%s match"%(comp, " do not" if should_match else "")) - # End for + ok, out = cprnc(comp, rfile1, rfile2, self._case, rundir2) # pylint: disable=unused-variable + logger.warning("CPRNC result for %s: %s"%(os.path.basename(rfile1), "PASS" if (ok == should_match) else "FAIL")) + compare_ok = compare_ok and (should_match == ok) + + expect(compare_ok, + "Not all restart files %s"%("matched" if should_match else "failed to match")) + diff --git a/scripts/lib/CIME/SystemTests/seq.py b/scripts/lib/CIME/SystemTests/seq.py index c53eb12a8643..3b6fd9477a8f 100644 --- a/scripts/lib/CIME/SystemTests/seq.py +++ b/scripts/lib/CIME/SystemTests/seq.py @@ -3,7 +3,7 @@ """ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.case_setup import case_setup, adjust_pio_layout +from CIME.case_setup import case_setup from CIME.check_lockedfiles import * import shutil @@ -90,7 +90,6 @@ def run_phase(self): # update the pelayout settings for this run self._case.read_xml() - adjust_pio_layout(self._case, self._case.get_value("PES_PER_NODE")) self.run_indv() @@ -103,7 +102,6 @@ def run_phase(self): logger.info("doing a second %d %s test with rootpes set to zero" % (stop_n, stop_option)) # update the pelayout settings for this run self._case.read_xml() - adjust_pio_layout(self._case, self._case.get_value("PES_PER_NODE")) self.run_indv(suffix="seq") self._component_compare_test("base", "seq") diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 04be59c613b9..a690386d4744 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -195,6 +195,11 @@ def run_indv(self, suffix="base", st_archive=False): stop_n = self._case.get_value("STOP_N") stop_option = self._case.get_value("STOP_OPTION") run_type = self._case.get_value("RUN_TYPE") + rundir = self._case.get_value("RUNDIR") + # remove any cprnc output leftover from previous runs + for compout in glob.iglob(os.path.join(rundir,"*.cprnc.out")): + os.remove(compout) + infostr = "doing an %d %s %s test" % (stop_n, stop_option, run_type) rest_option = self._case.get_value("REST_OPTION") @@ -360,7 +365,8 @@ def _compare_baseline(self): baselog = os.path.join(basecmp_dir, "cpl.log") if os.path.isfile(baselog) and len(memlist) > 3: - blmem = self._get_mem_usage(baselog)[-1][1] + blmem = self._get_mem_usage(baselog) + blmem = 0 if blmem == [] else blmem[-1][1] curmem = memlist[-1][1] if blmem != 0: diff = (curmem - blmem) / blmem diff --git a/scripts/lib/CIME/XML/compsets.py b/scripts/lib/CIME/XML/compsets.py index 875aea5d99a8..2d96aaed3e68 100644 --- a/scripts/lib/CIME/XML/compsets.py +++ b/scripts/lib/CIME/XML/compsets.py @@ -27,7 +27,9 @@ def get_compset_match(self, name): nodes = self.get_nodes("compset") alias = None lname = None + science_support = [] + for node in nodes: alias = self.get_element_text("alias",root=node) lname = self.get_element_text("lname",root=node) @@ -35,10 +37,14 @@ def get_compset_match(self, name): science_support_nodes = self.get_nodes("science_support", root=node) for node in science_support_nodes: science_support.append(node.get("grid")) - + user_mods_node = self.get_optional_node("user_mods", root=node) + if user_mods_node is not None: + user_mods = user_mods_node.text + else: + user_mods = None logger.debug("Found node match with alias: %s and lname: %s" % (alias, lname)) - return (lname, alias, science_support) - return (None, None, [False]) + return (lname, alias, science_support, user_mods) + return (None, None, [False], None) def get_compset_var_settings(self, compset, grid): ''' diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 88fcdfc8f210..6385be121b2b 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -7,7 +7,7 @@ from CIME.XML.standard_module_setup import * from CIME.utils import format_time from CIME.XML.env_base import EnvBase -from CIME.utils import transform_vars, get_cime_root +from CIME.utils import transform_vars, get_cime_root, convert_to_seconds from copy import deepcopy logger = logging.getLogger(__name__) @@ -181,7 +181,7 @@ def make_batch_script(self, input_template, job, case, total_tasks, tasks_per_no fd.write(output_text) os.chmod(job, os.stat(job).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=None): + def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=None, allow_walltime_override=False): if self.batchtype is None: self.batchtype = self.get_batch_system_type() @@ -195,18 +195,35 @@ def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=N else: task_count = int(task_count) - queue = force_queue if force_queue is not None else self.select_best_queue(task_count, job) - self.set_value("JOB_QUEUE", queue, subgroup=job) - - walltime = self.get_max_walltime(queue) if walltime is None else walltime - if walltime is None: - logger.warn("Could not find a queue matching task count %d, falling back to deprecated default walltime parameter"%task_count) - #if the user names a queue which is not defined in config_batch.xml and does not set a - #walltime, fall back to the max walltime in the default queue - if force_queue: - self.get_default_queue() - walltime = self._default_walltime + if force_queue: + if not self.queue_meets_spec(force_queue, task_count, walltime=walltime, job=job): + logger.warning("WARNING: User-requested queue '%s' does not meet requirements for job '%s'" % (force_queue, job)) + else: + queue = self.select_best_queue(task_count, walltime=walltime, job=job) + if queue is None and walltime is not None: + # Try to see if walltime was the holdup + queue = self.select_best_queue(task_count, walltime=None, job=job) + if queue is not None: + # It was, override the walltime if a test, otherwise just warn the user + new_walltime = self.get_queue_specs(queue)[3] + expect(new_walltime is not None, "Should never make it here") + logger.warning("WARNING: Requested walltime '%s' could not be matched by any queue" % walltime) + if allow_walltime_override: + logger.warning(" Using walltime '%s' instead" % new_walltime) + walltime = new_walltime + else: + logger.warning(" Continuing with suspect walltime, batch submission may fail") + + if queue is None: + logger.warning("WARNING: No queue on this system met the requirements for this job. Falling back to defaults") + default_queue_node = self.get_default_queue() + queue = default_queue_node.text + walltime = self.get_queue_specs(queue)[3] + + walltime = self.get_queue_specs(queue)[3] if walltime is None else walltime + walltime = self._default_walltime if walltime is None else walltime # last-chance fallback + self.set_value("JOB_QUEUE", queue, subgroup=job) self.set_value("JOB_WALLCLOCK_TIME", walltime, subgroup=job) logger.debug("Job %s queue %s walltime %s" % (job, queue, walltime)) @@ -281,7 +298,7 @@ def get_submit_args(self, case, job): return submitargs - def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): + def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=False): alljobs = self.get_jobs() startindex = 0 jobs = [] @@ -296,19 +313,23 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): continue try: prereq = self.get_value('prereq', subgroup=job, resolved=False) - if prereq is None or job == firstjob: + if prereq is None or job == firstjob or (dry_run and prereq == "$BUILD_COMPLETE"): prereq = True else: prereq = case.get_resolved_value(prereq) prereq = eval(prereq) except: expect(False,"Unable to evaluate prereq expression '%s' for job '%s'"%(self.get_value('prereq',subgroup=job), job)) + if prereq: - jobs.append((job,self.get_value('dependency', subgroup=job))) + jobs.append((job, self.get_value('dependency', subgroup=job))) + if self.batchtype == "cobalt": break depid = {} + jobcmds = [] + for job, dependency in jobs: if dependency is not None: deps = dependency.split() @@ -331,16 +352,21 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None): if slen == 0: jobid = None - logger.warn("job is %s"%job) - depid[job] = self.submit_single_job(case, job, jobid, no_batch=no_batch, batch_args=batch_args) + logger.warn("job is %s" % job) + result = self._submit_single_job(case, job, jobid, no_batch=no_batch, batch_args=batch_args, dry_run=dry_run) + batch_job_id = str(alljobs.index(job)) if dry_run else result + depid[job] = batch_job_id + jobcmds.append( (job, result) ) if self.batchtype == "cobalt": break - return sorted(list(depid.values())) + if dry_run: + return jobcmds + else: + return sorted(list(depid.values())) - def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=None): + def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=None, dry_run=False): logger.warn("Submit job %s"%job) - caseroot = case.get_value("CASEROOT") batch_system = self.get_value("BATCH_SYSTEM", subgroup=None) if batch_system is None or batch_system == "none" or no_batch: # Import here to avoid circular include @@ -351,20 +377,16 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No logger.info("Starting job script %s" % job) - # Hack until all testcases are ported to python - testcase = case.get_value("TESTCASE") - cimeroot = get_cime_root() - testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) - if job == "case.test" and testcase is not None and os.path.exists(testscript): - run_cmd_no_fail("%s --caseroot %s" % (os.path.join(".", job), caseroot)) - else: - # This is what we want longterm - function_name = job.replace(".", "_") + function_name = job.replace(".", "_") + if not dry_run: locals()[function_name](case) return submitargs = self.get_submit_args(case, job) + args_override = self.get_value("BATCH_COMMAND_FLAGS", subgroup=job) + if args_override: + submitargs = args_override if depid is not None: dep_string = self.get_value("depend_string", subgroup=None) @@ -383,11 +405,14 @@ def submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=No if string is not None: submitcmd += string + " " - logger.info("Submitting job script %s"%submitcmd) - output = run_cmd_no_fail(submitcmd, combine_output=True) - jobid = self.get_job_id(output) - logger.info("Submitted job id is %s"%jobid) - return jobid + if dry_run: + return submitcmd + else: + logger.info("Submitting job script %s"%submitcmd) + output = run_cmd_no_fail(submitcmd, combine_output=True) + jobid = self.get_job_id(output) + logger.info("Submitted job id is %s"%jobid) + return jobid def get_batch_system_type(self): nodes = self.get_nodes("batch_system") @@ -409,43 +434,71 @@ def get_job_id(self, output): jobid = search_match.group(1) return jobid - def select_best_queue(self, num_pes, job=None): + def queue_meets_spec(self, queue, num_pes, walltime=None, job=None): + jobmin, jobmax, jobname, walltimemax, strict = self.get_queue_specs(queue) + + # A job name match automatically meets spec + if job is not None and jobname is not None: + return jobname == job + + if jobmin is not None and num_pes < int(jobmin): + return False + + if jobmax is not None and num_pes > int(jobmax): + return False + + if walltime is not None and walltimemax is not None and strict: + walltime_s = convert_to_seconds(walltime) + walltimemax_s = convert_to_seconds(walltimemax) + if walltime_s > walltimemax_s: + return False + + return True + + def select_best_queue(self, num_pes, walltime=None, job=None): # Make sure to check default queue first. all_queues = [] all_queues.append( self.get_default_queue()) all_queues = all_queues + self.get_all_queues() for queue in all_queues: if queue is not None: - jobmin = queue.get("jobmin") - jobmax = queue.get("jobmax") - jobname = queue.get("jobname") - if jobname is not None: - if job == jobname: - return queue.text - # if the fullsum is between the min and max # jobs, then use this queue. - elif jobmin is not None and jobmax is not None and num_pes >= int(jobmin) and num_pes <= int(jobmax): - return queue.text + qname = queue.text + if self.queue_meets_spec(qname, num_pes, walltime=walltime, job=job): + return qname + return None - def get_max_walltime(self, queue): + def get_queue_specs(self, queue): + """ + Get queue specifications by name. + + Returns (jobmin, jobmax, jobname, walltimemax, is_strict) + """ for queue_node in self.get_all_queues(): if queue_node.text == queue: - return queue_node.get("walltimemax") + jobmin = queue_node.get("jobmin") + jobmax = queue_node.get("jobmax") + jobname = queue_node.get("jobname") + walltimemax = queue_node.get("walltimemax") + strict = queue_node.get("strict") == "true" + + return jobmin, jobmax, jobname, walltimemax, strict + + expect(False, "Queue '%s' is unknown to this system" % queue) def get_default_queue(self): node = self.get_optional_node("queue", attributes={"default" : "true"}) if node is None: node = self.get_optional_node("queue") expect(node is not None, "No queues found") - self._default_walltime = node.get("walltimemax") - return(node) + return node def get_all_queues(self): return self.get_nodes("queue") def get_nodes(self, nodename, attributes=None, root=None, xpath=None): if nodename in ("JOB_WALLCLOCK_TIME", "PROJECT", "PROJECT_REQUIRED", - "JOB_QUEUE"): + "JOB_QUEUE", "BATCH_COMMAND_FLAGS"): nodes = EnvBase.get_nodes(self, "entry", attributes={"id":nodename}, root=root, xpath=xpath) else: diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index b3efbb54bba9..e5d80ee4bdbb 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -13,12 +13,14 @@ # get_type) otherwise need to implement own functions and make GenericXML parent class class EnvMachSpecific(EnvBase): # pylint: disable=unused-argument - def __init__(self, caseroot, infile="env_mach_specific.xml",components=None): + def __init__(self, caseroot=None, infile="env_mach_specific.xml", + components=None, unit_testing=False): """ initialize an object interface to file env_mach_specific.xml in the case directory """ fullpath = infile if os.path.isabs(infile) else os.path.join(caseroot, infile) EnvBase.__init__(self, caseroot, fullpath) + self._unit_testing = unit_testing def populate(self, machobj): """Add entries to the file using information from a Machines object.""" @@ -191,6 +193,10 @@ def _match_attribs(self, attribs, compiler, debug, mpilib): elif ("debug" in attribs and not self._match("TRUE" if debug else "FALSE", attribs["debug"].upper())): return False + elif ("unit_testing" in attribs and + not self._match("TRUE" if self._unit_testing else "FALSE", + attribs["unit_testing"].upper())): + return False return True @@ -330,7 +336,7 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only best_num_matched = -1 default_match = None best_num_matched_default = -1 - args = {} + args = [] for mpirun_node in mpirun_nodes: xml_attribs = mpirun_node.attrib all_match = True @@ -382,7 +388,7 @@ def get_mpirun(self, case, attribs, check_members=None, job="case.run", exe_only subgroup=job, check_members=check_members, default=arg_node.get("default")) - args[arg_node.get("name")] = arg_value + args.append(arg_value) exec_node = self.get_node("executable", root=the_match) expect(exec_node is not None,"No executable found") diff --git a/scripts/lib/CIME/XML/env_test.py b/scripts/lib/CIME/XML/env_test.py index 44f947ec47f3..3c7461c74f44 100644 --- a/scripts/lib/CIME/XML/env_test.py +++ b/scripts/lib/CIME/XML/env_test.py @@ -103,3 +103,22 @@ def cleanupnode(self, node): if dnode is not None: node.remove(dnode) return node + + def set_value(self, vid, value, subgroup=None, ignore_type=False): + """ + check if vid is in test section of file + """ + newval = EnvBase.set_value(self, vid, value, subgroup, ignore_type) + if newval is None: + tnode = self.get_optional_node("test") + if tnode is not None: + newval = self.set_element_text(vid, value, root=tnode) + return newval + + def get_value(self, vid, attribute=None, resolved=True, subgroup=None): + value = EnvBase.get_value(self, vid, attribute, resolved, subgroup) + if value is None: + tnode = self.get_optional_node("test") + if tnode is not None: + value = self.get_element_text(vid, root=tnode) + return value diff --git a/scripts/lib/CIME/XML/files.py b/scripts/lib/CIME/XML/files.py index 61b2894b579d..a66721c6dfd4 100644 --- a/scripts/lib/CIME/XML/files.py +++ b/scripts/lib/CIME/XML/files.py @@ -20,6 +20,7 @@ def __init__(self): """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") + expect(os.path.isfile(infile), "Could not find or open file %s"%infile) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema) diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py index d40736d6e9b5..717e4b17bdd0 100644 --- a/scripts/lib/CIME/XML/generic_xml.py +++ b/scripts/lib/CIME/XML/generic_xml.py @@ -294,4 +294,5 @@ def get_raw_record(self, root=None): expect(False, "Could not write file %s, xml formatting error '%s'" % (self.filename, e)) return xmlstr - + def get_id(self): + return self.root.get("id") diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index cbe458e2afeb..5806045608ad 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -4,6 +4,7 @@ from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files +from CIME.utils import convert_to_unknown_type import socket @@ -191,6 +192,8 @@ def get_value(self, name, attributes=None, resolved=True, subgroup=None): elif name in os.environ: value = os.environ[name] + value = convert_to_unknown_type(value) + return value def get_field_from_list(self, listname, reqval=None, attributes=None): diff --git a/scripts/lib/CIME/XML/test_reporter.py b/scripts/lib/CIME/XML/test_reporter.py new file mode 100644 index 000000000000..261a13a9d14f --- /dev/null +++ b/scripts/lib/CIME/XML/test_reporter.py @@ -0,0 +1,113 @@ +""" +Interface to the testreporter xml. This class inherits from GenericXML.py + +""" + +from CIME.XML.standard_module_setup import * +from CIME.XML.generic_xml import GenericXML +from CIME.utils import expect,get_model + +import urllib + + + +class TestReporter(GenericXML): + + def __init__(self): + """ + initialize an object + """ + + expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." ) + self.root = None + + GenericXML.__init__(self) + + def setup_header(self, tagname,machine,compiler,mpilib,testroot,testtype,baseline): + # + # Create the XML header that the testdb is expecting to recieve + # + tlelem = ET.Element("testrecord") + elem = ET.Element('tag_name') + elem.text = tagname + tlelem.append(elem) + elem = ET.Element('mach') + elem.text = machine + tlelem.append(elem) + elem = ET.Element('compiler',attrib={"version":""}) + elem.text = compiler + tlelem.append(elem) + elem = ET.Element('mpilib',attrib={"version":""}) + elem.text = mpilib + tlelem.append(elem) + elem = ET.Element('testroot') + elem.text = testroot + tlelem.append(elem) + elem = ET.Element('testtype') + elem.text = testtype + tlelem.append(elem) + elem = ET.Element('baselinetag') + elem.text = baseline + tlelem.append(elem) + + self.root=tlelem + + + def add_result(self,test_name,test_status): + # + # Add a test result to the XML structure. + # + tlelem = ET.Element('tests',attrib={"testname":test_name}) + elem=ET.Element('category',attrib={"name":"casestatus"}) + tlelem.append(elem) + elem=ET.Element('category',attrib={"name":"comment"}) + elem.text= test_status['COMMENT'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"compare"}) + elem.text= test_status['BASELINE'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"memcomp"}) + elem.text= test_status['MEMCOMP'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"memleak"}) + elem.text= test_status['MEMLEAK'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"nlcomp"}) + elem.text= test_status['NLCOMP'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"status"}) + elem.text= test_status['STATUS'] + tlelem.append(elem) + + elem=ET.Element('category',attrib={"name":"tputcomp"}) + elem.text= test_status['TPUTCOMP'] + tlelem.append(elem) + + self.root.append(tlelem) + + + + def push2testdb(self): + # + # Post test result XML to CESM test database + # + xmlstr = ET.tostring(self.root,method="xml",encoding="UTF-8") + username=raw_input("Username:") + os.system("stty -echo") + password=raw_input("Password:") + os.system("stty echo") + params={'username':username,'password':password,'testXML':xmlstr} + url="https://csegweb.cgd.ucar.edu/testdb/cgi-bin/processXMLtest.cgi" + params = urllib.urlencode(params) + f = urllib.urlopen(url, params) + # + # Print any messages from the post command + # + print f.read() + print f.code + diff --git a/scripts/lib/CIME/aprun.py b/scripts/lib/CIME/aprun.py index 1e6169b4f0c7..9ce1aba46fdf 100755 --- a/scripts/lib/CIME/aprun.py +++ b/scripts/lib/CIME/aprun.py @@ -30,11 +30,10 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, >>> machine = "titan" >>> run_exe = "acme.exe" >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) - ('aprun -S 4 -n 680 -N 8 -d 2 acme.exe : -S 2 -n 128 -N 4 -d 4 acme.exe ', 117) - + (' -S 4 -n 680 -N 8 -d 2 acme.exe : -S 2 -n 128 -N 4 -d 4 acme.exe ', 117) >>> compiler = "intel" >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) - ('aprun -S 4 -cc numa_node -n 680 -N 8 -d 2 acme.exe : -S 2 -cc numa_node -n 128 -N 4 -d 4 acme.exe ', 117) + (' -S 4 -cc numa_node -n 680 -N 8 -d 2 acme.exe : -S 2 -cc numa_node -n 128 -N 4 -d 4 acme.exe ', 117) >>> ntasks = [64, 64, 64, 64, 64, 64, 64, 64, 1] >>> nthreads = [1, 1, 1, 1, 1, 1, 1, 1, 1] @@ -71,8 +70,8 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, maxt[c1] = 1 # Compute task and thread settings for batch commands - tasks_per_node, task_count, thread_count, max_thread_count, total_node_count, aprun = \ - 0, 1, maxt[0], maxt[0], 0, "aprun" + tasks_per_node, task_count, thread_count, max_thread_count, total_node_count, aprun_args = \ + 0, 1, maxt[0], maxt[0], 0, "" for c1 in xrange(1, total_tasks): if maxt[c1] != thread_count: tasks_per_node = min(pes_per_node, max_tasks_per_node / thread_count) @@ -83,11 +82,11 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, task_per_numa = int(math.ceil(tasks_per_node / 2.0)) # Option for Titan if machine == "titan" and tasks_per_node > 1: - aprun += " -S %d" % task_per_numa + aprun_args += " -S %d" % task_per_numa if compiler == "intel": - aprun += " -cc numa_node" + aprun_args += " -cc numa_node" - aprun += " -n %d -N %d -d %d %s :" % (task_count, tasks_per_node, thread_count, run_exe) + aprun_args += " -n %d -N %d -d %d %s :" % (task_count, tasks_per_node, thread_count, run_exe) node_count = int(math.ceil(float(task_count) / tasks_per_node)) total_node_count += node_count @@ -112,13 +111,13 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, # Special option for Titan with intel compiler if machine == "titan" and tasks_per_node > 1: - aprun += " -S %d" % task_per_numa + aprun_args += " -S %d" % task_per_numa if compiler == "intel": - aprun += " -cc numa_node" + aprun_args += " -cc numa_node" - aprun += " -n %d -N %d -d %d %s " % (task_count, tasks_per_node, thread_count, run_exe) + aprun_args += " -n %d -N %d -d %d %s " % (task_count, tasks_per_node, thread_count, run_exe) - return aprun, total_node_count + return aprun_args, total_node_count ############################################################################### def get_aprun_cmd_for_case(case, run_exe): diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index 74b34c2ef0f0..111c629f742a 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -301,6 +301,7 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro thread_bad_results, smp, compiler): ############################################################################### logger.info("Building %s with output to %s"%(compclass, file_build)) + t1 = time.time() with open(file_build, "w") as fd: stat = run_cmd("MODEL=%s SMP=%s %s/buildlib %s %s %s " % (compclass, stringify_bool(smp), config_dir, caseroot, libroot, bldroot), @@ -313,6 +314,8 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): shutil.copy(mod_file, incroot) + t2 = time.time() + logger.info("%s built in %f seconds" % (compclass, (t2 - t1))) ############################################################################### def _clean_impl(case, cleanlist, clean_all): diff --git a/scripts/lib/CIME/buildlib.py b/scripts/lib/CIME/buildlib.py index 33b4ffce9597..cc775e7bbbf1 100644 --- a/scripts/lib/CIME/buildlib.py +++ b/scripts/lib/CIME/buildlib.py @@ -3,7 +3,7 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import handle_standard_logging_options, setup_standard_logging_options +from CIME.utils import parse_args_and_handle_standard_logging_options, setup_standard_logging_options from CIME.case import Case import sys, os, argparse, doctest @@ -30,9 +30,7 @@ def parse_input(argv): parser.add_argument("bldroot", help="root for building library") - args = parser.parse_args() - - handle_standard_logging_options(args) + args = parse_args_and_handle_standard_logging_options(argv, parser) return args.caseroot, args.libroot, args.bldroot diff --git a/scripts/lib/CIME/buildnml.py b/scripts/lib/CIME/buildnml.py index 34f863155b67..0fa84fa578ee 100644 --- a/scripts/lib/CIME/buildnml.py +++ b/scripts/lib/CIME/buildnml.py @@ -5,7 +5,7 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import expect, handle_standard_logging_options, setup_standard_logging_options +from CIME.utils import expect, parse_args_and_handle_standard_logging_options, setup_standard_logging_options import sys, os, argparse, doctest logger = logging.getLogger(__name__) @@ -25,9 +25,7 @@ def parse_input(argv): parser.add_argument("caseroot", default=os.getcwd(), help="Case directory") - args = parser.parse_args() - - handle_standard_logging_options(args) + args = parse_args_and_handle_standard_logging_options(argv, parser) return args.caseroot diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index 01cd66de3569..15ec0f831712 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -30,6 +30,7 @@ from CIME.XML.env_run import EnvRun from CIME.XML.env_archive import EnvArchive from CIME.XML.env_batch import EnvBatch +from CIME.XML.generic_xml import GenericXML from CIME.user_mod_support import apply_user_mods from CIME.case_setup import case_setup from CIME.aprun import get_aprun_cmd_for_case @@ -95,7 +96,9 @@ def __init__(self, case_root=None, read_only=True): self._components = [] self._component_classes = [] self._is_env_loaded = False - + # these are user_mods as defined in the compset + # Command Line user_mods are handled seperately + self._user_mods = None self.thread_count = None self.total_tasks = None self.tasks_per_node = None @@ -105,7 +108,7 @@ def __init__(self, case_root=None, read_only=True): self.cores_per_task = None # check if case has been configured and if so initialize derived if self.get_value("CASEROOT") is not None: - self._initialize_derived_attributes() + self.initialize_derived_attributes() def check_if_comp_var(self, vid): vid = vid @@ -117,7 +120,7 @@ def check_if_comp_var(self, vid): return vid, comp, iscompvar return vid, comp, iscompvar - def _initialize_derived_attributes(self): + def initialize_derived_attributes(self): """ These are derived variables which can be used in the config_* files for variable substitution using the {{ var }} syntax @@ -147,7 +150,7 @@ def _initialize_derived_attributes(self): } executable = env_mach_spec.get_mpirun(self, mpi_attribs, job="case.run", exe_only=True)[0] - if executable == "aprun": + if executable is not None and "aprun" in executable: self.num_nodes = get_aprun_cmd_for_case(self, "acme.exe")[1] self.spare_nodes = env_mach_pes.get_spare_nodes(self.num_nodes) self.num_nodes += self.spare_nodes @@ -198,13 +201,14 @@ def get_case_root(self): """Returns the root directory for this case.""" return self._caseroot - def get_env(self, short_name): + def get_env(self, short_name, allow_missing=False): full_name = "env_%s.xml" % (short_name) for env_file in self._files: if os.path.basename(env_file.filename) == full_name: return env_file - - expect(False, "Could not find object for %s in case"%full_name) + if allow_missing: + return None + expect(False,"Could not find object for %s in case"%full_name) def copy(self, newcasename, newcaseroot, newcimeroot=None, newsrcroot=None): newcase = deepcopy(self) @@ -279,7 +283,8 @@ def get_value(self, item, attribute=None, resolved=True, subgroup=None): if resolved and type(result) is str: result = self.get_resolved_value(result) vtype = env_file.get_type_info(item) - result = convert_to_type(result, vtype, item) + if vtype is not None: + result = convert_to_type(result, vtype, item) return result for env_file in self._env_generic_files: @@ -368,6 +373,7 @@ def set_value(self, item, value, subgroup=None, ignore_type=False): if item == "CASEROOT": self._caseroot = value result = None + for env_file in self._env_entryid_files: result = env_file.set_value(item, value, subgroup, ignore_type) if (result is not None): @@ -419,7 +425,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf # If the file exists, read it and see if there is a match for the compset alias or longname if (os.path.isfile(compsets_filename)): compsets = Compsets(compsets_filename) - match, compset_alias, science_support = compsets.get_compset_match(name=compset_name) + match, compset_alias, science_support, self._user_mods = compsets.get_compset_match(name=compset_name) pesfile = files.get_value("PES_SPEC_FILE" , {"component":component}) if match is not None: self._pesfile = pesfile @@ -435,7 +441,10 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf self.set_lookup_value("USER_MODS_DIR" , user_mods_dir) self.set_lookup_value("PES_SPEC_FILE" , files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False)) - logger.info("Compset longname is %s " %(match)) + compset_info = "Compset longname is %s"%(match) + if self._user_mods is not None: + compset_info += " with user_mods directory %s"%(self._user_mods) + logger.info(compset_info) logger.info("Compset specification file is %s" %(compsets_filename)) logger.info("Pes specification file is %s" %(pesfile)) return compset_alias, science_support @@ -483,7 +492,7 @@ def __iter__(self): else: yield key, val - def _set_comp_classes(self, comp_classes): + def set_comp_classes(self, comp_classes): self._component_classes = comp_classes for env_file in self._env_entryid_files: env_file.set_components(comp_classes) @@ -512,7 +521,7 @@ def _get_component_config_data(self, files): # loop over all elements of both component_classes and components - and get config_component_file for # for each component - self._set_comp_classes(drv_comp.get_valid_model_components()) + self.set_comp_classes(drv_comp.get_valid_model_components()) if len(self._component_classes) > len(self._components): self._components.append('sesp') @@ -547,7 +556,8 @@ def configure(self, compset_name, grid_name, machine_name=None, project=None, pecount=None, compiler=None, mpilib=None, user_compset=False, pesfile=None, user_grid=False, gridfile=None, ninst=1, test=False, - walltime=None, queue=None, output_root=None, run_unsupported=False, answer=None): + walltime=None, queue=None, output_root=None, run_unsupported=False, answer=None, + input_dir=None): #-------------------------------------------- # compset, pesfile, and compset components @@ -585,8 +595,14 @@ def configure(self, compset_name, grid_name, machine_name=None, #-------------------------------------------- # set machine values in env_xxx files machobj = Machines(machine=machine_name) + probed_machine = machobj.probe_machine_name() machine_name = machobj.get_machine_name() - self.set_value("MACH",machine_name) + self.set_value("MACH", machine_name) + if probed_machine != machine_name and probed_machine is not None: + logger.warning("WARNING: User-selected machine '%s' does not match probed machine '%s'" % (machine_name, probed_machine)) + else: + logger.info("Machine is %s" % machine_name) + nodenames = machobj.get_node_names() nodenames = [x for x in nodenames if '_system' not in x and '_variables' not in x and 'mpirun' not in x and\ @@ -701,7 +717,12 @@ def configure(self, compset_name, grid_name, machine_name=None, if compclass == "CPL": continue key = "NINST_%s"%compclass - mach_pes_obj.set_value(key, ninst) + # ESP models are currently limited to 1 instance + if compclass == "ESP": + mach_pes_obj.set_value(key, 1) + else: + mach_pes_obj.set_value(key, ninst) + key = "NTASKS_%s"%compclass if key not in pes_ntasks.keys(): mach_pes_obj.set_value(key,1) @@ -720,7 +741,7 @@ def configure(self, compset_name, grid_name, machine_name=None, env_batch.set_batch_system(batch, batch_system_type=batch_system_type) env_batch.create_job_groups(bjobs) - env_batch.set_job_defaults(bjobs, pesize=maxval, walltime=walltime, force_queue=queue) + env_batch.set_job_defaults(bjobs, pesize=maxval, walltime=walltime, force_queue=queue, allow_walltime_override=test) self.schedule_rewrite(env_batch) #-------------------------------------------- @@ -789,10 +810,11 @@ def configure(self, compset_name, grid_name, machine_name=None, if model == "cesm" and not test: self.set_value("DOUT_S",True) self.set_value("TIMER_LEVEL", 4) + if test: self.set_value("TEST",True) - self._initialize_derived_attributes() + self.initialize_derived_attributes() # Make sure that parallel IO is not specified if total_tasks==1 if self.total_tasks == 1: @@ -805,6 +827,9 @@ def configure(self, compset_name, grid_name, machine_name=None, # Set TOTAL_CORES self.set_value("TOTAL_CORES", self.total_tasks * self.cores_per_task ) + if input_dir is not None: + self.set_value("DIN_LOC_ROOT", os.path.abspath(input_dir)) + def get_compset_var_settings(self): compset_obj = Compsets(infile=self.get_value("COMPSETS_SPEC_FILE")) matches = compset_obj.get_compset_var_settings(self._compsetname, self._gridname) @@ -844,6 +869,7 @@ def _create_caseroot_tools(self): os.path.join(toolsdir, "case.submit"), os.path.join(toolsdir, "case.cmpgen_namelists"), os.path.join(toolsdir, "preview_namelists"), + os.path.join(toolsdir, "preview_run"), os.path.join(toolsdir, "check_input_data"), os.path.join(toolsdir, "check_case"), os.path.join(toolsdir, "archive_metadata.sh"), @@ -933,7 +959,10 @@ def create_caseroot(self, clone=False): # Open a new README.case file in $self._caseroot append_status(" ".join(sys.argv), "README.case", caseroot=self._caseroot) - append_status("Compset longname is %s"%self.get_value("COMPSET"), + compset_info = "Compset longname is %s"%(self.get_value("COMPSET")) + if self._user_mods is not None: + compset_info += " with user_mods directory %s"%(self._user_mods) + append_status(compset_info, "README.case", caseroot=self._caseroot) append_status("Compset specification file is %s" % (self.get_value("COMPSETS_SPEC_FILE")), @@ -947,19 +976,35 @@ def create_caseroot(self, clone=False): comp_grid = "%s_GRID"%component_class append_status("%s is %s"%(comp_grid,self.get_value(comp_grid)), "README.case", caseroot=self._caseroot) + if self._user_mods is not None: + note = "This compset includes user_mods %s"%self._user_mods + append_status(note, "README.case", caseroot=self._caseroot) + logger.info(note) if not clone: self._create_caseroot_sourcemods() self._create_caseroot_tools() def apply_user_mods(self, user_mods_dir=None): - if user_mods_dir is not None: - if os.path.isabs(user_mods_dir): - user_mods_path = user_mods_dir - else: - user_mods_path = self.get_value('USER_MODS_DIR') - user_mods_path = os.path.join(user_mods_path, user_mods_dir) - self.set_value("USER_MODS_FULLPATH",user_mods_path) - apply_user_mods(self._caseroot, user_mods_path) + """ + User mods can be specified on the create_newcase command line (usually when called from create test) + or they can be in the compset definition, or both. + """ + + if self._user_mods is None: + compset_user_mods_resolved = None + else: + compset_user_mods_resolved = self.get_resolved_value(self._user_mods) + + # This looping order will lead to the specified user_mods_dir taking + # precedence over self._user_mods, if there are any conflicts. + for user_mods in (compset_user_mods_resolved, user_mods_dir): + if user_mods is not None: + if os.path.isabs(user_mods): + user_mods_path = user_mods + else: + user_mods_path = self.get_value('USER_MODS_DIR') + user_mods_path = os.path.join(user_mods_path, user_mods) + apply_user_mods(self._caseroot, user_mods_path) def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime_output_root=None): if cime_output_root is None: @@ -1062,9 +1107,9 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime return newcase - def submit_jobs(self, no_batch=False, job=None, batch_args=None): + def submit_jobs(self, no_batch=False, job=None, batch_args=None, dry_run=False): env_batch = self.get_env('batch') - return env_batch.submit_jobs(self, no_batch=no_batch, job=job, batch_args=batch_args) + return env_batch.submit_jobs(self, no_batch=no_batch, job=job, batch_args=batch_args, dry_run=dry_run) def get_mpirun_cmd(self, job="case.run"): env_mach_specific = self.get_env('mach_specific') @@ -1073,6 +1118,10 @@ def get_mpirun_cmd(self, job="case.run"): run_misc_suffix = "" if run_misc_suffix is None else run_misc_suffix run_suffix = run_exe + run_misc_suffix + mpirun_cmd_override = self.get_value("MPI_RUN_COMMAND") + if mpirun_cmd_override not in ["", None, "UNSET"]: + return mpirun_cmd_override + " " + run_exe + " " + run_misc_suffix + # Things that will have to be matched against mpirun element attributes mpi_attribs = { "compiler" : self.get_value("COMPILER"), @@ -1081,20 +1130,21 @@ def get_mpirun_cmd(self, job="case.run"): "unit_testing" : False } - executable, args = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) + executable, mpi_arg_list = env_mach_specific.get_mpirun(self, mpi_attribs, job=job) # special case for aprun - if executable == "aprun": - aprun_cmd, num_nodes = get_aprun_cmd_for_case(self, run_exe) + if executable is not None and "aprun" in executable: + aprun_args, num_nodes = get_aprun_cmd_for_case(self, run_exe) expect( (num_nodes + self.spare_nodes) == self.num_nodes, "Not using optimized num nodes") - return aprun_cmd + " " + run_misc_suffix + return executable + aprun_args + " " + run_misc_suffix + else: - mpi_arg_string = " ".join(args.values()) + mpi_arg_string = " ".join(mpi_arg_list) - if self.get_value("BATCH_SYSTEM") == "cobalt": - mpi_arg_string += " : " + if self.get_value("BATCH_SYSTEM") == "cobalt": + mpi_arg_string += " : " - return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) + return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, run_suffix) def set_model_version(self, model): version = "unknown" @@ -1153,3 +1203,50 @@ def _check_testlists(self, compset_alias, grid_name, files): expect(False, "\nThis compset and grid combination is untested in CESM. " "Override this warning with the --run-unsupported option to create_newcase.", error_prefix="STOP: ") + + def set_file(self, xmlfile): + """ + force the case object to consider only xmlfile + """ + expect(os.path.isfile(xmlfile), "Could not find file %s"%xmlfile) + + gfile = GenericXML(infile=xmlfile) + ftype = gfile.get_id() + + logger.warn("setting case file to %s"%xmlfile) + new_env_file = None + for env_file in self._env_entryid_files: + if os.path.basename(env_file.filename) == ftype: + if ftype == "env_run.xml": + new_env_file = EnvRun(infile=xmlfile) + elif ftype == "env_build.xml": + new_env_file = EnvBuild(infile=xmlfile) + elif ftype == "env_case.xml": + new_env_file = EnvCase(infile=xmlfile) + elif ftype == "env_mach_pes.xml": + new_env_file = EnvMachPes(infile=xmlfile) + elif ftype == "env_batch.xml": + new_env_file = EnvBatch(infile=xmlfile) + elif ftype == "env_test.xml": + new_env_file = EnvTest(infile=xmlfile) + if new_env_file is not None: + self._env_entryid_files = [] + self._env_generic_files = [] + self._env_entryid_files.append(new_env_file) + break + if new_env_file is None: + for env_file in self._env_generic_files: + if os.path.basename(env_file.filename) == ftype: + if ftype == "env_archive.xml": + new_env_file = EnvArchive(infile=xmlfile) + elif ftype == "env_mach_specific.xml": + new_env_file = EnvMachSpecific(infile=xmlfile) + else: + expect(False, "No match found for file type %s"%ftype) + if new_env_file is not None: + self._env_entryid_files = [] + self._env_generic_files = [] + self._env_generic_files.append(new_env_file) + break + + self._files = self._env_entryid_files + self._env_generic_files diff --git a/scripts/lib/CIME/case_run.py b/scripts/lib/CIME/case_run.py index 0b4728b2f466..31ff36d00f91 100644 --- a/scripts/lib/CIME/case_run.py +++ b/scripts/lib/CIME/case_run.py @@ -245,6 +245,10 @@ def case_run(case): "Please submit your run using the submit script like so:" " ./case.submit") + # Forces user to use case.submit if they re-submit + if case.get_value("TESTCASE") is None: + case.set_value("RUN_WITH_SUBMIT", False) + prerun_script = case.get_value("PRERUN_SCRIPT") postrun_script = case.get_value("POSTRUN_SCRIPT") diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py index 5ab6383373d2..a4e158d7116c 100644 --- a/scripts/lib/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -59,7 +59,7 @@ def _build_usernl_files(case, model, comp): shutil.copy(model_nl, nlfile) ############################################################################### -def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, adjust_pio=True): +def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): ############################################################################### os.chdir(caseroot) @@ -127,17 +127,22 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, continue ninst = case.get_value("NINST_%s" % comp) ntasks = case.get_value("NTASKS_%s" % comp) + # ESP models are currently limited to 1 instance + expect((comp != "ESP") or (ninst == 1), + "ESP components may only have one instance") if ninst > ntasks: if ntasks == 1: case.set_value("NTASKS_%s" % comp, ninst) else: expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks)) - # Set TOTAL_CORES - case.set_value("TOTAL_CORES", case.total_tasks * case.cores_per_task ) - if os.path.exists("case.run"): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") + + case.initialize_derived_attributes() + + # Set TOTAL_CORES + case.set_value("TOTAL_CORES", case.total_tasks * case.cores_per_task ) else: check_pelayouts_require_rebuild(case, models) @@ -150,32 +155,29 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, logger.debug("at update TOTALPES = %s"%pestot) case.set_value("TOTALPES", pestot) thread_count = env_mach_pes.get_max_thread_count(models) - build_threaded = case.get_build_threaded() - expect(not (build_threaded and compiler == "nag"), - "it is not possible to run with OpenMP if using the NAG Fortran compiler") cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH")) case.set_value("COST_PES", cost_pes) + # Make sure pio settings are consistent + tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count) + + case.initialize_derived_attributes() + # create batch files logger.info("Creating batch script case.run") env_batch = case.get_env("batch") num_nodes = case.num_nodes - tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count) for job in env_batch.get_jobs(): input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job)) if job == "case.test" and testcase is not None and not test_mode: logger.info("Writing %s script" % job) - testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) - # Short term fix to be removed when csh tests are removed - if not os.path.exists(testscript): - env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) + env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) elif job != "case.test": logger.info("Writing %s script from input template %s" % (job, input_batch_script)) env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) - # Make sure pio settings are consistant - if adjust_pio: - adjust_pio_layout(case, tasks_per_node) + # Set TOTAL_CORES + case.set_value("TOTAL_CORES", case.total_tasks * case.cores_per_task ) # Make a copy of env_mach_pes.xml in order to be able # to check that it does not change once case.setup is invoked @@ -214,29 +216,12 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh") env_module.save_all_env_info("software_environment.txt") -def adjust_pio_layout(case, new_pio_stride): - - models = case.get_values("COMP_CLASSES") - for comp in models: - pio_stride = case.get_value("PIO_STRIDE_%s"%comp) - pio_numtasks = case.get_value("PIO_NUMTASKS_%s"%comp) - ntasks = case.get_value("NTASKS_%s"%comp) - new_stride = min(ntasks, new_pio_stride) - new_numtasks = max(1, ntasks//new_stride) - if pio_stride != new_stride: - logger.info("Resetting PIO_STRIDE_%s to %s"%(comp, new_stride)) - case.set_value("PIO_STRIDE_%s"%comp, new_stride) - if pio_numtasks != new_numtasks: - logger.info("Resetting PIO_NUMTASKS_%s to %s"%(comp, new_numtasks)) - case.set_value("PIO_NUMTASKS_%s"%comp, new_numtasks) - - ############################################################################### -def case_setup(case, clean=False, test_mode=False, reset=False, adjust_pio=True): +def case_setup(case, clean=False, test_mode=False, reset=False): ############################################################################### caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID") phase = "setup.clean" if clean else "case.setup" - functor = lambda: _case_setup_impl(case, caseroot, clean, test_mode, reset, adjust_pio) + functor = lambda: _case_setup_impl(case, caseroot, clean, test_mode, reset) if case.get_value("TEST") and not test_mode: test_name = casebaseid if casebaseid is not None else case.get_value("CASE") diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 992cd23cf916..a76fb0051f24 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -53,11 +53,11 @@ def _get_ninst_info(case, compclass): ninst_strings = [] if ninst is None: ninst = 1 - for i in range(ninst): - if ninst > 1: - ninst_strings.append('_' + '%04d' % i) - else: - ninst_strings.append('') + for i in range(1,ninst+1): + if ninst > 1: + ninst_strings.append('_' + '%04d' % i) + else: + ninst_strings.append('') logger.debug("ninst and ninst_strings are: %s and %s for %s" %(ninst, ninst_strings, compclass)) return ninst, ninst_strings @@ -255,7 +255,7 @@ def _archive_restarts(case, archive, archive_entry, restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] else: pattern = r"%s\.%s\d*.*" % (casename, compname) - if pattern != "dart": + if "dart" not in pattern: pfile = re.compile(pattern) files = [f for f in os.listdir(rundir) if pfile.search(f)] if ninst_strings: diff --git a/scripts/lib/CIME/case_test.py b/scripts/lib/CIME/case_test.py index 85aaed7149ab..2ea51caa1d0d 100644 --- a/scripts/lib/CIME/case_test.py +++ b/scripts/lib/CIME/case_test.py @@ -66,4 +66,6 @@ def case_test(case, testname=None): success = test.run() + case.set_value("RUN_WITH_SUBMIT", False) + return success diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py index ee99a37398b0..312437143888 100644 --- a/scripts/lib/CIME/check_lockedfiles.py +++ b/scripts/lib/CIME/check_lockedfiles.py @@ -88,7 +88,7 @@ def check_lockedfiles(caseroot=None): for lfile in lockedfiles: fpart = os.path.basename(lfile) # ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name - if lfile.count('.') > 1: + if fpart.count('.') > 1: continue cfile = os.path.join(caseroot, fpart) if os.path.isfile(cfile): diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 03d1c6cabfa0..8d8b5a966705 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -26,7 +26,7 @@ def _run_pylint(on_file, interactive): cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\",\"%s\"))'"%\ (os.path.join(cimeroot,"scripts","lib"), os.path.join(cimeroot,"scripts","Tools"), - os.path.join(cimeroot,"tools","unit_testing","python")) + os.path.join(cimeroot,"scripts","fortran_unit_testing","python")) cmd = "%s %s %s" % (pylint, cmd_options, on_file) logger.debug("pylint command is %s"%cmd) diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index d91a31184c32..5e2ed18ab4db 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -987,9 +987,12 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): >>> x.set_variable_value('foo', 'bar(2)', [u'3'], var_size=4) >>> x.get_variable_value('foo', 'bar') [u'1', u'3'] - >>> x.set_variable_value('foo', 'bar', [u'2']) + >>> x.set_variable_value('foo', 'bar(1)', [u'2']) >>> x.get_variable_value('foo', 'bar') [u'2', u'3'] + >>> x.set_variable_value('foo', 'bar', [u'1']) + >>> x.get_variable_value('foo', 'bar') + [u'1', u'3'] >>> x.set_variable_value('foo', 'bazz', [u'3']) >>> x.set_variable_value('Brack', 'baR', [u'4']) >>> x.get_variable_value('foo', 'bazz') @@ -1534,9 +1537,11 @@ def _parse_variable_name(self, allow_equals=True): Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: '' is not a valid variable name + >>> _NamelistParser('foo+= ')._parse_variable_name() + u'foo' """ old_pos = self._pos - separators = (' ', '\n', '=') if allow_equals else (' ', '\n') + separators = (' ', '\n', '=', '+') if allow_equals else (' ', '\n') while self._curr() not in separators: self._advance() text = self._text[old_pos:self._pos] @@ -1659,6 +1664,29 @@ def _look_ahead_for_equals(self, pos): break return False + def _look_ahead_for_plusequals(self, pos): + r"""Look ahead to see if the next two non-whitespace character are '+='. + + The `pos` argument is the position in the text to start from while + looking. This function returns a boolean. + + >>> _NamelistParser('+=')._look_ahead_for_plusequals(0) + True + >>> _NamelistParser('a \n+=')._look_ahead_for_plusequals(1) + True + >>> _NamelistParser('')._look_ahead_for_plusequals(0) + False + >>> _NamelistParser('a+=')._look_ahead_for_plusequals(0) + False + """ + for test_pos in range(pos, self._len): + if self._text[test_pos] not in (' ', '\n'): + if self._text[test_pos] == '+': + return self._look_ahead_for_equals(test_pos + 1) + else: + break + return False + def _parse_literal(self, allow_name=False, allow_eof_end=False): r"""Parse and return a variable value at the current position. @@ -1734,16 +1762,28 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' + >>> _NamelistParser('foo+= ')._parse_literal() + Traceback (most recent call last): + ... + _NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo+=' >>> _NamelistParser('5,')._parse_literal(allow_name=True) u'5' >>> x = _NamelistParser('foo= ') >>> x._parse_literal(allow_name=True) >>> x._curr() u'f' + >>> x = _NamelistParser('foo+= ') + >>> x._parse_literal(allow_name=True) + >>> x._curr() + u'f' >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' + >>> _NamelistParser('6*foo+= ')._parse_literal(allow_name=True) + Traceback (most recent call last): + ... + _NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo+=' >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() @@ -1782,6 +1822,7 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): separators = [' ', '\n', ',', '/'] if allow_name: separators.append('=') + separators.append('+') while new_pos != self._len and self._text[new_pos] not in separators: # allow commas if they are inside () if self._text[new_pos] == '(': @@ -1794,9 +1835,12 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): # At the end of the file, give up by throwing an EOF. self._advance(self._len) # If `allow_name` is set, we need to check and see if the next non-blank - # character is '=', and return `None` if so. + # character is '=' or the next two are '+=', and return `None` if so. if allow_name and self._look_ahead_for_equals(new_pos): return + elif allow_name and self._look_ahead_for_plusequals(new_pos): + return + self._advance(new_pos - self._pos, check_eof=allow_eof_end) text = self._text[old_pos:self._pos] if not any(is_valid_fortran_namelist_literal(type_, text) @@ -1911,7 +1955,8 @@ def _parse_name_and_values(self, allow_eof_end=False): r"""Parse and return a variable name and values assigned to that name. The return value of this function is a tuple containing (a) the name of - the variable in a string, and (b) a list of the variable's values. Null + the variable in a string, (b) a list of the variable's values, and + (c) whether or not to add the found value to existing variable. Null values are represented by the empty string. If `allow_eof_end=True`, the end of the sequence of values might come @@ -1919,55 +1964,62 @@ def _parse_name_and_values(self, allow_eof_end=False): alternate file format in "groupless" mode.) >>> _NamelistParser("foo='bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"]) + (u'foo', [u"'bar'"], False) >>> _NamelistParser("foo(3)='bar' /")._parse_name_and_values() - (u'foo(3)', [u"'bar'"]) + (u'foo(3)', [u"'bar'"], False) >>> _NamelistParser("foo ='bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"]) + (u'foo', [u"'bar'"], False) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() - (u'foo', [u"'bar'"]) + (u'foo', [u"'bar'"], False) >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected '=' but found "'" >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"]) + (u'foo', [u"'bar'", u"'bazz'"], False) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() - (u'foo', [u'', u'', u"'bazz'", u'6*']) + (u'foo', [u'', u'', u"'bazz'", u'6*'], False) >>> _NamelistParser("foo='bar' 'bazz' foo2='ban'")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"]) + (u'foo', [u"'bar'", u"'bazz'"], False) >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() - (u'foo', [u"'bar'", u"'bazz'"]) + (u'foo', [u"'bar'", u"'bazz'"], False) >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() Traceback (most recent call last): ... _NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'', u'', u"'bazz'", u'6*']) + (u'foo', [u'', u'', u"'bazz'", u'6*'], False) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) - (u'foo(3)', [u"'bazz'"]) + (u'foo(3)', [u"'bazz'"], False) >>> _NamelistParser("foo=")._parse_name_and_values() Traceback (most recent call last): ... _NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'']) + (u'foo', [u''], False) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'']) + (u'foo', [u''], False) >>> _NamelistParser("foo=2")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'2']) + (u'foo', [u'2'], False) >>> _NamelistParser("foo=1,2")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'1', u'2']) + (u'foo', [u'1', u'2'], False) >>> _NamelistParser("foo(1:2)=1,2,3 ")._parse_name_and_values(allow_eof_end=True) Traceback (most recent call last): ... SystemExit: ERROR: Too many values for array foo(1:2) >>> _NamelistParser("foo=1,")._parse_name_and_values(allow_eof_end=True) - (u'foo', [u'1', u'']) + (u'foo', [u'1', u''], False) + >>> _NamelistParser("foo+=1")._parse_name_and_values(allow_eof_end=True) + (u'foo', [u'1'], True) """ name = self._parse_variable_name() + addto = False # This keeps track of whether += existed self._eat_whitespace() + # check to see if we have a "+=" + if self._curr() == '+': + self._advance() + addto=True # tell parser that we want to add to dictionary values self._expect_char("=") try: self._advance() @@ -1975,7 +2027,7 @@ def _parse_name_and_values(self, allow_eof_end=False): except _NamelistEOF: # If we hit the end of file, return a name assigned to a null value. if allow_eof_end: - return name, [u''] + return name, [u''], addto else: raise # Expect at least one literal, even if it's a null value. @@ -1994,7 +2046,7 @@ def _parse_name_and_values(self, allow_eof_end=False): arraylen =max(0,1 + ((maxindex - minindex)/step)) expect(len(values) <= arraylen, "Too many values for array %s"%(name)) - return name, values + return name, values, addto def _parse_namelist_group(self): r"""Parse an entire namelist group, adding info to `self._settings`. @@ -2037,6 +2089,22 @@ def _parse_namelist_group(self): >>> x._parse_namelist_group() >>> x._settings OrderedDict([(u'foo', [u"'bar'"])]) + >>> x = _NamelistParser("&group foo='bar', foo+='baz' /", groupless=True) + >>> x._parse_namelist_group() + >>> x._settings + OrderedDict([(u'foo', [u"'bar'", u"'baz'"])]) + >>> x = _NamelistParser("&group foo+='bar' /", groupless=True) + >>> x._parse_namelist_group() + >>> x._settings + OrderedDict([(u'foo', [u"'bar'"])]) + >>> x = _NamelistParser("&group foo='bar', foo+='baz' /") + >>> x._parse_namelist_group() + >>> x._settings + OrderedDict([(u'group', {u'foo': [u"'bar'", u"'baz'"]})]) + >>> x = _NamelistParser("&group foo+='bar' /") + >>> x._parse_namelist_group() + >>> x._settings + OrderedDict([(u'group', {u'foo': [u"'bar'"]})]) """ group_name = self._parse_namelist_group_name() if not self._groupless: @@ -2047,18 +2115,24 @@ def _parse_namelist_group(self): self._settings[group_name] = {} self._eat_whitespace() while self._curr() != '/': - name, values = self._parse_name_and_values() + name, values, addto = self._parse_name_and_values() dsettings = [] if self._groupless: if name in self._settings: dsettings = self._settings[name] - values = merge_literal_lists(dsettings, values) + if addto: + values = self._settings[name] + values + if not addto: + values = merge_literal_lists(dsettings, values) self._settings[name] = values else: group = self._settings[group_name] if name in group: dsettings = group[name] - values = merge_literal_lists(dsettings, values) + if addto: + values = group[name] + values + if not addto: + values = merge_literal_lists(dsettings, values) group[name] = values def parse_namelist(self): @@ -2085,16 +2159,24 @@ def parse_namelist(self): OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u'']), (u'foo2', [u'2*5', u'6'])]) >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() OrderedDict([(u'foo', [u"'bar'"])]) - >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bazz'"])]) - >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() - OrderedDict([(u'foo', [u"'bar'"])]) >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() OrderedDict([(u'foo', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() OrderedDict([(u'foo(2)', [u"'bar'"])]) >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() OrderedDict([(u'foo(2)', [u"'bar'"]), (u'foo(3)', [u"'bazz'"])]) + >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bazz'"])]) + >>> _NamelistParser("foo='bar'\n foo+='bazz'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bar'", u"'bazz'"])]) + >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bazz'"])]) + >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bar'"])]) + >>> _NamelistParser("foo='bar', 'bazz'\n foo+='ban'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bar'", u"'bazz'", u"'ban'"])]) + >>> _NamelistParser("foo+='bar'", groupless=True).parse_namelist() + OrderedDict([(u'foo', [u"'bar'"])]) """ # Return empty dictionary for empty files. if self._len == 0: @@ -2108,9 +2190,12 @@ def parse_namelist(self): # Handle case with no namelist groups. if self._groupless and self._curr() != '&': while self._pos < self._len: - name, values = self._parse_name_and_values(allow_eof_end=True) + name, values, addto = self._parse_name_and_values(allow_eof_end=True) if name in self._settings: - values = merge_literal_lists(self._settings[name], values) + if addto: + values = self._settings[name] + values + else: + values = merge_literal_lists(self._settings[name], values) self._settings[name] = values return self._settings # Loop over namelist groups in the file. diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py index 06dfd81e26fa..b29d788b0bf5 100644 --- a/scripts/lib/CIME/nmlgen.py +++ b/scripts/lib/CIME/nmlgen.py @@ -326,7 +326,10 @@ def _sub_fields(self, varnames): if not line: continue if "%glc" in line: - glc_nec_indices = range(self._case.get_value('GLC_NEC')) + if self._case.get_value('GLC_NEC') == 0: + glc_nec_indices = [0] + else: + glc_nec_indices = range(self._case.get_value('GLC_NEC')) glc_nec_indices.append(glc_nec_indices[-1] + 1) glc_nec_indices.pop(0) for i in glc_nec_indices: diff --git a/scripts/lib/CIME/preview_namelists.py b/scripts/lib/CIME/preview_namelists.py index 3f1429809af3..ce98d2447ad7 100644 --- a/scripts/lib/CIME/preview_namelists.py +++ b/scripts/lib/CIME/preview_namelists.py @@ -40,7 +40,7 @@ def create_dirs(case): with open(os.path.join(dir_,"CASEROOT"),"w+") as fd: fd.write(caseroot+"\n") -def create_namelists(case): +def create_namelists(case, component=None): """ Create component namelists """ @@ -73,36 +73,37 @@ def create_namelists(case): else: compname = case.get_value("COMP_%s" % model_str.upper()) - cmd = os.path.join(config_dir, "buildnml") - do_run_cmd = False - # This code will try to import and run each buildnml as a subroutine - # if that fails it will run it as a program in a seperate shell - try: - with open(cmd, 'r') as f: - first_line = f.readline() - if "python" in first_line: - mod = imp.load_source("buildnml", cmd) - logger.info(" Calling %s buildnml"%compname) - mod.buildnml(case, caseroot, compname) - else: - raise SyntaxError - except SyntaxError as detail: - if 'python' in first_line: - expect(False, detail) - else: + if component is None or component == model_str: + cmd = os.path.join(config_dir, "buildnml") + do_run_cmd = False + # This code will try to import and run each buildnml as a subroutine + # if that fails it will run it as a program in a seperate shell + try: + with open(cmd, 'r') as f: + first_line = f.readline() + if "python" in first_line: + mod = imp.load_source("buildnml", cmd) + logger.info(" Calling %s buildnml"%compname) + mod.buildnml(case, caseroot, compname) + else: + raise SyntaxError + except SyntaxError as detail: + if 'python' in first_line: + expect(False, detail) + else: + do_run_cmd = True + except AttributeError: do_run_cmd = True - except AttributeError: - do_run_cmd = True - except: - raise - - if do_run_cmd: - logger.info(" Running %s buildnml"%compname) - case.flush() - output = run_cmd_no_fail("%s %s" % (cmd, caseroot), verbose=False) - logger.info(output) - # refresh case xml object from file - case.read_xml() + except: + raise + + if do_run_cmd: + logger.info(" Running %s buildnml"%compname) + case.flush() + output = run_cmd_no_fail("%s %s" % (cmd, caseroot), verbose=False) + logger.info(output) + # refresh case xml object from file + case.read_xml() logger.info("Finished creating component namelists") diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index b3f977fc988f..a884e17550da 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -8,7 +8,7 @@ they can be run outside the context of TestScheduler. """ -import shutil, traceback, stat, threading, time, glob +import traceback, stat, threading, time, glob from CIME.XML.standard_module_setup import * import CIME.compare_namelists import CIME.utils @@ -118,6 +118,9 @@ def __init__(self, test_names, test_data=None, else: self._project = project + # Needed in case default root depends on PROJECT + self._machobj.set_value("PROJECT", project) + # We will not use batch system if user asked for no_batch or if current # machine is not a batch machine self._no_batch = no_batch or not self._machobj.has_batch_system() @@ -147,7 +150,7 @@ def __init__(self, test_names, test_data=None, if parallel_jobs is None: self._parallel_jobs = min(len(test_names), - int(self._machobj.get_value("MAX_TASKS_PER_NODE"))) + self._machobj.get_value("MAX_TASKS_PER_NODE")) else: self._parallel_jobs = parallel_jobs @@ -325,28 +328,26 @@ def _update_test_status(self, test, phase, status): def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None): ########################################################################### while True: - rc, output, _ = run_cmd(cmd, combine_output=True, from_dir=from_dir) + rc, output, errput = run_cmd(cmd, from_dir=from_dir) if rc != 0: self._log_output(test, "%s FAILED for test '%s'.\nCommand: %s\nOutput: %s\n" % - (phase, test, cmd, output)) + (phase, test, cmd, output + "\n" + errput)) # Temporary hack to get around odd file descriptor use by # buildnml scripts. if "bad interpreter" in output: time.sleep(1) continue else: - break + return False, errput else: # We don't want "RUN PASSED" in the TestStatus.log if the only thing that # succeeded was the submission. phase = "SUBMIT" if phase == RUN_PHASE else phase self._log_output(test, "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s\n" % - (phase, test, cmd, output)) - break - - return rc == 0 + (phase, test, cmd, output + "\n" + errput)) + return True, errput ########################################################################### def _create_newcase_phase(self, test): @@ -364,6 +365,8 @@ def _create_newcase_phase(self, test): create_newcase_cmd += " --project %s " % self._project if self._output_root is not None: create_newcase_cmd += " --output-root %s " % self._output_root + if self._input_dir is not None: + create_newcase_cmd += " --input-dir %s " % self._input_dir if test_mods is not None: files = Files() @@ -376,9 +379,12 @@ def _create_newcase_phase(self, test): testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) if not os.path.exists(test_mod_file): - self._log_output(test, "Missing testmod file '%s'" % test_mod_file) - return False + error = "Missing testmod file '%s'" % test_mod_file + self._log_output(test, error) + return False, error + create_newcase_cmd += " --user-mods-dir %s" % test_mod_file + mpilib = None if case_opts is not None: for case_opt in case_opts: # pylint: disable=not-an-iterable @@ -399,7 +405,6 @@ def _create_newcase_phase(self, test): create_newcase_cmd += " --mpilib %s" % self._mpilib logger.debug (" MPILIB set to %s" % self._mpilib) - if self._queue is not None: create_newcase_cmd += " --queue=%s" % self._queue @@ -533,10 +538,7 @@ def _xml_phase(self, test): case.set_value("TEST", True) case.set_value("SAVE_TIMING", self._save_timing) - if self._input_dir is not None: - case.set_value("DIN_LOC_ROOT", self._input_dir) - - return True + return True, "" ########################################################################### def _setup_phase(self, test): @@ -545,8 +547,9 @@ def _setup_phase(self, test): rv = self._shell_cmd_for_phase(test, "./case.setup", SETUP_PHASE, from_dir=test_dir) # It's OK for this command to fail with baseline diffs but not catastrophically - cmdstat, output, _ = run_cmd("./case.cmpgen_namelists", combine_output=True, from_dir=test_dir) - expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % output) + if rv[0]: + cmdstat, output, _ = run_cmd("./case.cmpgen_namelists", combine_output=True, from_dir=test_dir) + expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % output) return rv @@ -580,11 +583,10 @@ def _run_catch_exceptions(self, test, phase, run): return run(test) except (SystemExit, Exception) as e: exc_tb = sys.exc_info()[2] - errput = "Test '%s' failed in phase '%s' with exception '%s'" % (test, phase, str(e)) + errput = "Test '%s' failed in phase '%s' with exception '%s'\n" % (test, phase, str(e)) + errput += ''.join(traceback.format_tb(exc_tb)) self._log_output(test, errput) - logger.warning("Caught exception: %s" % str(e)) - traceback.print_tb(exc_tb) - return False + return False, errput ########################################################################### def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False): @@ -641,7 +643,7 @@ def _update_test_status_file(self, test, test_phase, status): def _consumer(self, test, test_phase, phase_method): ########################################################################### before_time = time.time() - success = self._run_catch_exceptions(test, test_phase, phase_method) + success, errors = self._run_catch_exceptions(test, test_phase, phase_method) elapsed_time = time.time() - before_time status = (TEST_PEND_STATUS if test_phase == RUN_PHASE and not \ self._no_batch else TEST_PASS_STATUS) if success else TEST_FAIL_STATUS @@ -659,7 +661,9 @@ def _consumer(self, test, test_phase, phase_method): (test_phase, test, elapsed_time, status) if not success: - status_str += " Case dir: %s" % self._get_test_dir(test) + status_str += "\n Case dir: %s\n" % self._get_test_dir(test) + status_str += " Errors were:\n %s\n" % "\n ".join(errors.splitlines()) + logger.info(status_str) if test_phase in [CREATE_NEWCASE_PHASE, XML_PHASE]: @@ -742,7 +746,8 @@ def _setup_cs_files(self): template = open(template_file, "r").read() template = template.replace("", os.path.join(self._cime_root,"scripts","Tools")).replace\ - ("", self._test_id) + ("", self._test_id).replace\ + ("", self._test_root) if not os.path.exists(self._test_root): os.makedirs(self._test_root) cs_status_file = os.path.join(self._test_root, "cs.status.%s" % self._test_id) @@ -767,11 +772,16 @@ def _setup_cs_files(self): os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) - if get_model == "cesm": - testreporter = os.path.join(self._test_root,"testreporter.pl") - shutil.copy(os.path.join(self._cime_root,"scripts","Testing","testreporter.pl"), - testreporter) - os.chmod(testreporter, os.stat(testreporter).st_mode | stat.S_IXUSR | stat.S_IXGRP) + if get_model() == "cesm": + template_file = os.path.join(python_libs_root, "testreporter.template") + template = open(template_file, "r").read() + template = template.replace("", + os.path.join(self._cime_root, "scripts", "Tools")) + testreporter_file = os.path.join(self._test_root, "testreporter") + with open(testreporter_file, "w") as fd: + fd.write(template) + os.chmod(testreporter_file, os.stat(testreporter_file).st_mode + | stat.S_IXUSR | stat.S_IXGRP) except Exception as e: logger.warning("FAILED to set up cs files: %s" % str(e)) diff --git a/scripts/lib/CIME/test_status.py b/scripts/lib/CIME/test_status.py index ed4071cd7ea3..7c955b7e0898 100644 --- a/scripts/lib/CIME/test_status.py +++ b/scripts/lib/CIME/test_status.py @@ -209,24 +209,26 @@ def get_status(self, phase): def get_comment(self, phase): return self._phase_statuses[phase][1] if phase in self._phase_statuses else None - def phase_statuses_dump(self, fd, prefix=''): + def phase_statuses_dump(self, prefix=''): """ Args: - fd: file open for writing prefix: string printed at the start of each line """ + result = "" if self._phase_statuses: for phase, data in self._phase_statuses.iteritems(): status, comments = data if not comments: - fd.write("%s%s %s %s\n" % (prefix, status, self._test_name, phase)) + result += "%s%s %s %s\n" % (prefix, status, self._test_name, phase) else: - fd.write("%s%s %s %s %s\n" % (prefix, status, self._test_name, phase, comments)) + result += "%s%s %s %s %s\n" % (prefix, status, self._test_name, phase, comments) + + return result def flush(self): if self._phase_statuses and not self._no_io: with open(self._filename, "w") as fd: - self.phase_statuses_dump(fd) + fd.write(self.phase_statuses_dump()) def _parse_test_status(self, file_contents): """ diff --git a/scripts/lib/CIME/tests/test_user_mod_support.py b/scripts/lib/CIME/tests/test_user_mod_support.py new file mode 100644 index 000000000000..c8e582f01b17 --- /dev/null +++ b/scripts/lib/CIME/tests/test_user_mod_support.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +import unittest +import shutil +import tempfile +import os +from CIME.user_mod_support import apply_user_mods + +# ======================================================================== +# Define some parameters +# ======================================================================== + +_SOURCEMODS = os.path.join("SourceMods", "src.drv") + +class TestUserModSupport(unittest.TestCase): + + # ======================================================================== + # Test helper functions + # ======================================================================== + + def setUp(self): + self._caseroot = tempfile.mkdtemp() + self._caseroot_sourcemods = os.path.join(self._caseroot, _SOURCEMODS) + os.makedirs(self._caseroot_sourcemods) + self._user_mods_parent_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._caseroot, ignore_errors=True) + shutil.rmtree(self._user_mods_parent_dir, ignore_errors=True) + + def createUserMod(self, name, include_dirs=None): + """Create a user_mods directory with the given name. + + This directory is created within self._user_mods_parent_dir + + For name='foo', it will contain: + + - A user_nl_cpl file with contents: + foo + + - A shell_commands file with contents: + echo foo >> /PATH/TO/CASEROOT/shell_commands_result + + - A file in _SOURCEMODS named myfile.F90 with contents: + foo + + If include_dirs is given, it should be a list of strings, giving names + of other user_mods directories to include. e.g., if include_dirs is + ['foo1', 'foo2'], then this will create a file 'include_user_mods' that + contains paths to the 'foo1' and 'foo2' user_mods directories, one per + line. + """ + + mod_dir = os.path.join(self._user_mods_parent_dir, name) + os.makedirs(mod_dir) + mod_dir_sourcemods = os.path.join(mod_dir, _SOURCEMODS) + os.makedirs(mod_dir_sourcemods) + + with open(os.path.join(mod_dir, "user_nl_cpl"), "w") as user_nl_cpl: + user_nl_cpl.write(name + "\n") + with open(os.path.join(mod_dir, "shell_commands"), "w") as shell_commands: + command = "echo %s >> %s/shell_commands_result\n"%(name, + self._caseroot) + shell_commands.write(command) + with open(os.path.join(mod_dir_sourcemods, "myfile.F90"), "w") as f90_file: + f90_file.write(name + "\n") + + if include_dirs: + with open(os.path.join(mod_dir, "include_user_mods"), "w") as include_user_mods: + for one_include in include_dirs: + include_user_mods.write(os.path.join(self._user_mods_parent_dir, one_include) + "\n") + + def assertResults(self, expected_user_nl_cpl, + expected_shell_commands_result, + expected_sourcemod, + msg = ""): + """Asserts that the contents of the files in self._caseroot match expectations + + If msg is provided, it is printed for some failing assertions + """ + + path_to_user_nl_cpl = os.path.join(self._caseroot, "user_nl_cpl") + self.assertTrue(os.path.isfile(path_to_user_nl_cpl), + msg = msg + ": user_nl_cpl does not exist") + with open(path_to_user_nl_cpl, "r") as user_nl_cpl: + contents = user_nl_cpl.read() + self.assertEqual(expected_user_nl_cpl, contents) + + path_to_shell_commands_result = os.path.join(self._caseroot, "shell_commands_result") + self.assertTrue(os.path.isfile(path_to_shell_commands_result), + msg = msg + ": shell_commands_result does not exist") + with open(path_to_shell_commands_result, "r") as shell_commands_result: + contents = shell_commands_result.read() + self.assertEqual(expected_shell_commands_result, contents) + + path_to_sourcemod = os.path.join(self._caseroot_sourcemods, "myfile.F90") + self.assertTrue(os.path.isfile(path_to_sourcemod), + msg = msg + ": sourcemod file does not exist") + with open(path_to_sourcemod, "r") as sourcemod: + contents = sourcemod.read() + self.assertEqual(expected_sourcemod, contents) + + # ======================================================================== + # Begin actual tests + # ======================================================================== + + def test_basic(self): + self.createUserMod("foo") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo")) + self.assertResults(expected_user_nl_cpl = "foo\n", + expected_shell_commands_result = "foo\n", + expected_sourcemod = "foo\n", + msg = "test_basic") + + def test_two_applications(self): + """If apply_user_mods is called twice, the second should appear after the first so that it takes precedence.""" + + self.createUserMod("foo1") + self.createUserMod("foo2") + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo1")) + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "foo2")) + self.assertResults(expected_user_nl_cpl = "foo1\nfoo2\n", + expected_shell_commands_result = "foo1\nfoo2\n", + expected_sourcemod = "foo2\n", + msg = "test_two_applications") + + def test_include(self): + """If there is an included mod, the main one should appear after the included one so that it takes precedence.""" + + self.createUserMod("base") + self.createUserMod("derived", include_dirs=["base"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived")) + + self.assertResults(expected_user_nl_cpl = "base\nderived\n", + expected_shell_commands_result = "base\nderived\n", + expected_sourcemod = "derived\n", + msg = "test_include") + + def test_duplicate_includes(self): + """Test multiple includes, where both include the same base mod. + + The base mod should only be included once. + """ + + self.createUserMod("base") + self.createUserMod("derived1", include_dirs=["base"]) + self.createUserMod("derived2", include_dirs=["base"]) + self.createUserMod("derived_combo", + include_dirs = ["derived1", "derived2"]) + + apply_user_mods(self._caseroot, + os.path.join(self._user_mods_parent_dir, "derived_combo")) + + # NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not + # critical here: If this aspect of the behavior changes, the + # expected_contents can be changed to match the new behavior in this + # respect. + expected_contents = """base +derived2 +derived1 +derived_combo +""" + self.assertResults(expected_user_nl_cpl = expected_contents, + expected_shell_commands_result = expected_contents, + expected_sourcemod = "derived_combo\n", + msg = "test_duplicate_includes") diff --git a/scripts/lib/CIME/user_mod_support.py b/scripts/lib/CIME/user_mod_support.py index 572910e43b22..d27c7255542d 100644 --- a/scripts/lib/CIME/user_mod_support.py +++ b/scripts/lib/CIME/user_mod_support.py @@ -14,6 +14,9 @@ def apply_user_mods(caseroot, user_mods_path): updating SourceMods and creating case shell_commands and xmlchange_cmds files First remove case shell_commands files if any already exist + + If this function is called multiple times, settings from later calls will + take precedence over earlier calls, if there are conflicts. ''' case_shell_command_files = [os.path.join(caseroot,"shell_commands"), os.path.join(caseroot,"xmlchange_cmnds")] @@ -22,6 +25,13 @@ def apply_user_mods(caseroot, user_mods_path): os.remove(shell_command_file) include_dirs = build_include_dirs_list(user_mods_path) + # If a user_mods dir 'foo' includes 'bar', the include_dirs list returned + # from build_include_dirs has 'foo' before 'bar'. But with the below code, + # directories that occur later in the list take precedence over the earlier + # ones, and we want 'foo' to take precedence over 'bar' in this case (in + # general: we want a given user_mods directory to take precedence over any + # mods that it includes). So we reverse include_dirs to accomplish this. + include_dirs.reverse() logger.debug("include_dirs are %s"%include_dirs) for include_dir in include_dirs: # write user_nl_xxx file in caseroot @@ -31,7 +41,11 @@ def apply_user_mods(caseroot, user_mods_path): if len(newcontents) == 0: continue case_user_nl = user_nl.replace(include_dir, caseroot) - update_user_nl_file(case_user_nl, newcontents) + # If the same variable is set twice in a user_nl file, the later one + # takes precedence. So by appending the new contents, later entries + # in the include_dirs list take precedence over earlier entries. + with open(case_user_nl, "a") as fd: + fd.write(newcontents) # update SourceMods in caseroot for root, _, files in os.walk(include_dir,followlinks=True,topdown=False): @@ -39,21 +53,18 @@ def apply_user_mods(caseroot, user_mods_path): for sfile in files: source_mods = os.path.join(root,sfile) case_source_mods = source_mods.replace(include_dir, caseroot) + # We overwrite any existing SourceMods file so that later + # include_dirs take precedence over earlier ones if os.path.isfile(case_source_mods): - logger.warn("Refusing to overwrite existing SourceMods in %s"%case_source_mods) + logger.warn("WARNING: Overwriting existing SourceMods in %s"%case_source_mods) else: logger.info("Adding SourceMod to case %s"%case_source_mods) - try: - shutil.copyfile(source_mods, case_source_mods) - except: - expect(False, "Could not write file %s in caseroot %s" - %(case_source_mods,caseroot)) + try: + shutil.copyfile(source_mods, case_source_mods) + except: + expect(False, "Could not write file %s in caseroot %s" + %(case_source_mods,caseroot)) - # Reverse include_dirs to make sure xmlchange commands are called in the - # correct order; it may be desireable to reverse include_dirs above the - # previous loop and then append user_nl changes rather than prepend them. - include_dirs.reverse() - for include_dir in include_dirs: # create xmlchange_cmnds and shell_commands in caseroot shell_command_files = glob.glob(os.path.join(include_dir,"shell_commands")) +\ glob.glob(os.path.join(include_dir,"xmlchange_cmnds")) @@ -70,6 +81,8 @@ def apply_user_mods(caseroot, user_mods_path): shell_commands_file) with open(shell_commands_file,"r") as fd: new_shell_commands = fd.read().replace("xmlchange","xmlchange --force") + # By appending the new commands to the end, settings from later + # include_dirs take precedence over earlier ones with open(case_shell_commands, "a") as fd: fd.write(new_shell_commands) @@ -78,16 +91,6 @@ def apply_user_mods(caseroot, user_mods_path): os.chmod(shell_command_file, 0777) run_cmd_no_fail(shell_command_file) -def update_user_nl_file(case_user_nl, contents): - if os.path.isfile(case_user_nl): - with open(case_user_nl, "r") as fd: - old_contents = fd.read() - contents = contents + old_contents - logger.debug("Pre-pending file %s"%(case_user_nl)) - with open(case_user_nl, "w") as fd: - fd.write(contents) - - def build_include_dirs_list(user_mods_path, include_dirs=None): ''' diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 9216c85ff142..57a28addbe31 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -45,7 +45,7 @@ def check_name(fullname, additional_chars=None, fullpath=False): True """ - chars = '<>/{}[\]~`@' # pylint: disable=anomalous-backslash-in-string + chars = '<>/{}[\]~`@:' # pylint: disable=anomalous-backslash-in-string if additional_chars is not None: chars += additional_chars if fullpath: @@ -259,7 +259,7 @@ def run_cmd_no_fail(cmd, input_str=None, from_dir=None, verbose=None, if stat != 0: # If command produced no errput, put output in the exception since we # have nothing else to go on. - errput = output if errput == "" else errput + errput = output if not errput else errput expect(False, "Command: '%s' failed with error '%s'%s" % (cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir)) @@ -637,7 +637,7 @@ def filter(self, record): #non-zero return means we log this message return 1 if record.levelno < self.max_level else 0 -def handle_standard_logging_options(args): +def parse_args_and_handle_standard_logging_options(args, parser=None): """ Guide to logging in CIME. @@ -660,6 +660,11 @@ def handle_standard_logging_options(args): stderr_stream_handler = logging.StreamHandler(stream=sys.stderr) stderr_stream_handler.setLevel(logging.WARNING) + # scripts_regression_tests is the only thing that should pass a None argument in parser + if parser is not None: + _check_for_invalid_args(args[1:]) + args = parser.parse_args(args[1:]) + # --verbose adds to the message format but does not impact the log level if args.verbose: stdout_stream_handler.setFormatter(verbose_formatter) @@ -682,6 +687,8 @@ def handle_standard_logging_options(args): root_logger.setLevel(logging.WARN) else: root_logger.setLevel(logging.INFO) + return args + def get_logging_options(): """ @@ -714,9 +721,9 @@ def convert_to_type(value, type_str, vid=""): expect(False, "Entry %s was listed as type int but value '%s' is not valid int" % (vid, value)) elif type_str == "logical": - expect(value in ["TRUE", "FALSE","true","false"], + expect(value.upper() in ["TRUE", "FALSE"], "Entry %s was listed as type logical but had val '%s' instead of TRUE or FALSE" % (vid, value)) - value = value == "TRUE" or value == "true" + value = value.upper() == "TRUE" elif type_str == "real": try: @@ -729,6 +736,36 @@ def convert_to_type(value, type_str, vid=""): return value +def convert_to_unknown_type(value): + """ + Convert value to it's real type by probing conversions. + """ + if value is not None: + + # Attempt to convert to logical + if value.upper() in ["TRUE", "FALSE"]: + return value.upper() == "TRUE" + + # Attempt to convert to integer + try: + value = int(eval(value)) + except: + pass + else: + return value + + # Attempt to convert to float + try: + value = float(value) + except: + pass + else: + return value + + # Just treat as string + + return value + def convert_to_string(value, type_str=None, vid=""): """ Convert value back to string. @@ -787,6 +824,25 @@ def convert_to_babylonian_time(seconds): return "%02d:%02d:%02d" % (hours, minutes, seconds) +def get_time_in_seconds(timeval, unit): + """ + Convert a time from 'unit' to seconds + """ + if 'nyear' in unit: + dmult = 365 * 24 * 3600 + elif 'nmonth' in unit: + dmult = 30 * 24 * 3600 + elif 'nday' in unit: + dmult = 24 * 3600 + elif 'nhour' in unit: + dmult = 3600 + elif 'nminute' in unit: + dmult = 60 + else: + dmult = 1 + + return dmult * timeval + def compute_total_time(job_cost_map, proc_pool): """ Given a map: jobname -> (procs, est-time), return a total time @@ -1172,7 +1228,7 @@ def run_and_log_case_status(func, phase, caseroot='.'): try: rv = func() except: - e = sys.exc_info()[0] + e = sys.exc_info()[1] append_case_status(phase, "error", msg=("\n%s" % e), caseroot=caseroot) raise else: @@ -1180,6 +1236,17 @@ def run_and_log_case_status(func, phase, caseroot='.'): return rv +def _check_for_invalid_args(args): + for arg in args: + if arg.startswith("--"): + continue + if arg.startswith("-") and len(arg) > 2: +# Uncomment these lines when we want to enforce --mulitchararg syntax +# if arg == "-value" or arg == "-noecho": + logger.warn("This argument is depricated, please use -%s"%arg) +# else: +# expect(False, "Invalid argument %s\n Multi-character arguments should begin with \"--\" and single character with \"-\"\n Use --help for a complete list of available options"%arg) + class SharedArea(object): """ Enable 0002 umask within this manager diff --git a/scripts/lib/cs.status.template b/scripts/lib/cs.status.template index f774bcc03e19..25f7f0fb2e80 100644 --- a/scripts/lib/cs.status.template +++ b/scripts/lib/cs.status.template @@ -1,3 +1,3 @@ #! /bin/bash -/cs_status *./TestStatus +/cs.status "$@" /*./TestStatus diff --git a/scripts/lib/testreporter.template b/scripts/lib/testreporter.template new file mode 100644 index 000000000000..c8bfa79405a2 --- /dev/null +++ b/scripts/lib/testreporter.template @@ -0,0 +1,3 @@ +#! /bin/bash + +/testreporter.py "$@" diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py index cf1885f3fb08..ae22c63f8b8c 100644 --- a/scripts/lib/update_acme_tests.py +++ b/scripts/lib/update_acme_tests.py @@ -48,8 +48,9 @@ "ERP.f45_g37_rx1.A", "SMS_D_Ln9.f19_g16_rx1.A", "DAE.f19_f19.A", - "SMS.T42_T42.S") -# "PRE.f19_f19.ADESP") + "PET_P32.f19_f19.A", + "SMS.T42_T42.S", + "PRE.f45_g37_rx1.ADESP") ), # diff --git a/scripts/manage_case b/scripts/manage_case index c561d6cede48..40f6a8fc280e 100755 --- a/scripts/manage_case +++ b/scripts/manage_case @@ -140,9 +140,7 @@ def parse_command_line(args): parser.add_argument("--long", action="store_true", help="Provide long output for queries") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args diff --git a/scripts/manage_pes b/scripts/manage_pes index 6258b76e6d6c..9f0f86e52cfa 100755 --- a/scripts/manage_pes +++ b/scripts/manage_pes @@ -98,8 +98,8 @@ def parse_command_line(args, description): help=queryhelp) parser.add_argument("-machine", "--machine", default=None, help="can be a supported machine name") - args = parser.parse_args(args[1:]) - CIME.utils.handle_standard_logging_options(args) + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) CIME.utils.expect(args.add or args.query, "Either --query or --add must be on command line") diff --git a/scripts/query_testlists b/scripts/query_testlists index 12d3fc0c3eb8..d9c5f3a31d3d 100755 --- a/scripts/query_testlists +++ b/scripts/query_testlists @@ -11,7 +11,7 @@ from CIME.utils import expect logger = logging.getLogger(__name__) ############################################################################### -def parse_command_line(description): +def parse_command_line(args, description): ############################################################################### parser = argparse.ArgumentParser( description=description) @@ -45,9 +45,7 @@ def parse_command_line(description): parser.add_argument("--xml-testlist", help="Use this testlist to lookup tests, default specified in config_files.xml") - args = parser.parse_args() - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) expect(not(args.count and args.list_type), "Cannot specify both --count and --list arguments.") @@ -149,7 +147,7 @@ def list_test_data(test_data, list_type): ############################################################################### def _main_func(description): ############################################################################### - args = parse_command_line(description) + args = parse_command_line(sys.argv, description) test_data = get_tests_from_xml( xml_machine = args.xml_machine, diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 762bf7bda702..f94eafd30b4c 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -178,17 +178,25 @@ def setUpClass(cls): cls._testroot = os.path.join(TEST_ROOT, 'TestUnitTests') cls._testdirs = [] + def _has_unit_test_support(self): + default_compiler = MACHINE.get_default_compiler() + compiler = Compilers(MACHINE, compiler=default_compiler) + attrs = {'MPILIB': 'mpi-serial', 'compile_threaded': 'false'} + pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH", + attributes=attrs) + if pfunit_path is None: + return False + else: + return True + def test_a_unit_test(self): cls = self.__class__ - machine = MACHINE.get_machine_name() - compiler = MACHINE.get_default_compiler() - if (machine != "yellowstone" or compiler != "intel"): - #TODO: get rid of this restriction - self.skipTest("Skipping TestUnitTest - only supported on yellowstone with intel") + if not self._has_unit_test_support(): + self.skipTest("Skipping TestUnitTest - PFUNIT_PATH not found for the default compiler on this machine") test_dir = os.path.join(cls._testroot,"unit_tester_test") cls._testdirs.append(test_dir) os.makedirs(test_dir) - unit_test_tool = os.path.abspath(os.path.join(CIME.utils.get_cime_root(),"tools","unit_testing","run_tests.py")) + unit_test_tool = os.path.abspath(os.path.join(CIME.utils.get_cime_root(),"scripts","fortran_unit_testing","run_tests.py")) test_spec_dir = os.path.join(os.path.dirname(unit_test_tool),"Examples", "interpolate_1d", "tests") run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s"\ %(unit_test_tool,test_dir,test_spec_dir)) @@ -199,17 +207,14 @@ def test_b_cime_f90_unit_tests(self): if (FAST_ONLY): self.skipTest("Skipping slow test") - machine = MACHINE.get_machine_name() - compiler = MACHINE.get_default_compiler() - if (machine != "yellowstone" or compiler != "intel"): - #TODO: get rid of this restriction - self.skipTest("Skipping TestUnitTest - only supported on yellowstone with intel") + if not self._has_unit_test_support(): + self.skipTest("Skipping TestUnitTest - PFUNIT_PATH not found for the default compiler on this machine") test_dir = os.path.join(cls._testroot,"driver_f90_tests") cls._testdirs.append(test_dir) os.makedirs(test_dir) test_spec_dir = CIME.utils.get_cime_root() - unit_test_tool = os.path.abspath(os.path.join(test_spec_dir,"tools","unit_testing","run_tests.py")) + unit_test_tool = os.path.abspath(os.path.join(test_spec_dir,"scripts","fortran_unit_testing","run_tests.py")) run_cmd_no_fail("%s --build-dir %s --test-spec-dir %s"\ %(unit_test_tool,test_dir,test_spec_dir)) @@ -248,7 +253,10 @@ def test_a_createnewcase(self): shutil.rmtree(testdir) cls._testdirs.append(testdir) - run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --output-root %s" % (SCRIPT_DIR, testdir, cls._testroot), from_dir=SCRIPT_DIR) + run_cmd_assert_result(self, "%s/create_newcase --case CreateNewcaseTest --script-root %s --compset X --res f19_g16 --output-root %s" % (SCRIPT_DIR, testdir, cls._testroot), from_dir=SCRIPT_DIR) + self.assertTrue(os.path.exists(testdir)) + self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup"))) + run_cmd_assert_result(self, "./case.setup", from_dir=testdir) run_cmd_assert_result(self, "./case.build", from_dir=testdir) @@ -1268,6 +1276,9 @@ def test_cime_case(self): run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s" % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT)) + self.assertEqual(type(MACHINE.get_value("MAX_TASKS_PER_NODE")), int) + self.assertTrue(type(MACHINE.get_value("PROJECT_REQUIRED")) in [type(None) , bool]) + casedir = os.path.join(self._testroot, "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name)) self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) @@ -1376,6 +1387,108 @@ def test_cime_case_force_pecount(self): expected_cores = 16 * case.cores_per_task self.assertEqual(case.get_value("TOTAL_CORES"), expected_cores) + ########################################################################### + def test_cime_case_xmlchange_append(self): + ########################################################################### + run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --force-procs 16 --force-threads 8" + % (SCRIPT_DIR, self._baseline_name, self._testroot, self._testroot)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_Mmpi-serial_P16x8.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + run_cmd_assert_result(self, "./xmlchange --id PIO_CONFIG_OPTS --val='-opt1'", from_dir=casedir) + result = run_cmd_assert_result(self, "./xmlquery --value PIO_CONFIG_OPTS", from_dir=casedir) + self.assertEqual(result, "-opt1") + + run_cmd_assert_result(self, "./xmlchange --id PIO_CONFIG_OPTS --val='-opt2' --append", from_dir=casedir) + result = run_cmd_assert_result(self, "./xmlquery --value PIO_CONFIG_OPTS", from_dir=casedir) + self.assertEqual(result, "-opt1 -opt2") + + ########################################################################### + def test_cime_case_test_walltime_mgmt_1(self): + ########################################################################### + if CIME.utils.get_model() != "acme": + self.skipTest("Skipping walltime test. Depends on ACME batch settings") + + test_name = "ERS.f19_g16_rx1.A" + machine, compiler = "blues", "gnu" + run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s" % + (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + result = run_cmd_assert_result(self, "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "0:10:00") + + result = run_cmd_assert_result(self, "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "shared") + + ########################################################################### + def test_cime_case_test_walltime_mgmt_2(self): + ########################################################################### + if CIME.utils.get_model() != "acme": + self.skipTest("Skipping walltime test. Depends on ACME batch settings") + + test_name = "ERS_P64.f19_g16_rx1.A" + machine, compiler = "blues", "gnu" + run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s" % + (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + result = run_cmd_assert_result(self, "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "03:00:00") + + result = run_cmd_assert_result(self, "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "batch") + + ########################################################################### + def test_cime_case_test_walltime_mgmt_3(self): + ########################################################################### + if CIME.utils.get_model() != "acme": + self.skipTest("Skipping walltime test. Depends on ACME batch settings") + + test_name = "ERS_P64.f19_g16_rx1.A" + machine, compiler = "blues", "gnu" + run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s --walltime='0:10:00'" % + (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + result = run_cmd_assert_result(self, "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "0:10:00") + + result = run_cmd_assert_result(self, "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "batch") # Not smart enough to select faster queue + + ########################################################################### + def test_cime_case_test_walltime_mgmt_4(self): + ########################################################################### + if CIME.utils.get_model() != "acme": + self.skipTest("Skipping walltime test. Depends on ACME batch settings") + + test_name = "ERS_P1.f19_g16_rx1.A" + machine, compiler = "blues", "gnu" + run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s --walltime='2:00:00'" % + (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot)) + + casedir = os.path.join(self._testroot, + "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name)) + self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir) + + result = run_cmd_assert_result(self, "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "01:00:00") + + result = run_cmd_assert_result(self, "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) + self.assertEqual(result, "shared") + ############################################################################### class X_TestSingleSubmit(TestCreateTestCommon): ############################################################################### @@ -2191,7 +2304,7 @@ def _main_func(): else: setattr(args, log_param, False) - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, None) write_provenance_info() @@ -2215,7 +2328,8 @@ def _main_func(): print "All pass, removing directory:", TEST_ROOT if os.path.exists(TEST_ROOT): shutil.rmtree(TEST_ROOT) - raise + + raise if (__name__ == "__main__"): _main_func() diff --git a/src/build_scripts/buildlib.gptl b/src/build_scripts/buildlib.gptl index 2a5fea6f8528..b92c788d396d 100755 --- a/src/build_scripts/buildlib.gptl +++ b/src/build_scripts/buildlib.gptl @@ -6,8 +6,8 @@ cd $CASEROOT # CASEROOT is always assumed to be an environment variable -set CIMEROOT = `./xmlquery CIMEROOT -value ` -set GMAKE = `./xmlquery GMAKE -value ` +set CIMEROOT = `./xmlquery CIMEROOT -value ` +set GMAKE = `./xmlquery GMAKE --value ` # NOTE- (mv, 2015-01-02) SHAREDPATH is an environment variable set in # the $CASE.build script diff --git a/src/build_scripts/buildlib.pio b/src/build_scripts/buildlib.pio index cb4d29e40bf7..5b6909b0ddfb 100755 --- a/src/build_scripts/buildlib.pio +++ b/src/build_scripts/buildlib.pio @@ -38,9 +38,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("caseroot", nargs="?", default=os.getcwd(), help="Case directory to build") - args = parser.parse_args(args[1:]) - - CIME.utils.handle_standard_logging_options(args) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) return args.buildroot, args.installpath, args.caseroot @@ -144,7 +142,7 @@ def _main_func(description): case.set_valid_values("PIO_TYPENAME",valid_values) # nothing means use the general default valid_values += ",nothing" - + for comp in case.get_values("COMP_CLASSES"): comp_pio_typename = "%s_PIO_TYPENAME"%comp case.set_valid_values(comp_pio_typename,valid_values) diff --git a/src/components/data_comps/datm/cime_config/config_component.xml b/src/components/data_comps/datm/cime_config/config_component.xml index f5a32e93253f..888066ca2bda 100644 --- a/src/components/data_comps/datm/cime_config/config_component.xml +++ b/src/components/data_comps/datm/cime_config/config_component.xml @@ -15,27 +15,22 @@ char - CORE2_NYF,CORE2_IAF,TN460,CLM_QIAN,CLM_QIAN_WISO,CLM1PT,CLMCRUNCEP,CLMCRUNCEP_V5,CLMGSWP3,CPLHIST3HrWx,COPYALL_NPS_v1,COPYALL_NPS_CORE2_v1,WRF,WW3,CPLHISTForcingForOcnIce + CORE2_NYF,CORE2_IAF,CLM_QIAN,CLM_QIAN_WISO,CLM1PT,CLMCRUNCEP,CLMCRUNCEP_V5,CLMGSWP3,CPLHISTForcing CORE2_NYF run_component_datm env_run.xml Mode for data atmosphere component. - The default is CORE2_NYF (CORE2 normal year forcing) is the - DATM mode used in C and G compsets. CLM_QIAN, CLMCRUNCEP, CLMGSWP3 and CLM1PT are - modes using observational data for forcing CLM in I compsets. + CORE2_NYF (CORE2 normal year forcing) is the DATM mode used in C and G compsets. + CLM_QIAN, CLMCRUNCEP, CLMGSWP3 and CLM1PT are modes using observational data for I compsets. - CORE2_NYF - CORE2_IAF - WW3 - COPYALL_NPS_v1 - COPYALL_NPS_CORE2_v1 + CORE2_NYF + CORE2_IAF CLM_QIAN_WISO - CLM_QIAN - CLMCRUNCEP - CLMGSWP3 - CLM1PT - CPLHIST3HrWx - CPLHISTForcingForOcnIce + CLM_QIAN + CLMCRUNCEP + CLMGSWP3 + CLM1PT + CPLHISTForcing @@ -58,7 +53,7 @@ none none none - cplhist + cplhist run_component_datm env_run.xml @@ -279,8 +274,6 @@ single point tower site atm input data: COREv2 datm normal year forcing: (requires additional user-supplied data) COREv2 datm interannual year forcing: (requires additional user-supplied data) - WW3 wave watch data from a short period of hi temporal frequency COREv2 data - DATM NPS forcing: diff --git a/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml b/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml index f5b666c66521..006a3f363d78 100644 --- a/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +++ b/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml @@ -31,22 +31,20 @@ Currently the following streams are supported (term definitions precede the streams) - CLM_QIAN = Run with the CLM Qian dataset valid from 1948 to 2004 (force CLM) - CLM_QIAN_WISO = Run with the CLM Qian dataset with isotopes valid from 2000 to 2004 (force CLM) - CLMCRUNCEP = Run with the CLM CRU NCEP V4 ( default ) forcing valid from 1900 to 2010 (force CLM) - CLMCRUNCEP_V5 = Run with the CLM CRU NCEP V5 forcing valid from 1900 to 2010 (force CLM) - CLMGSWP3 = Run with the CLM GSWP3 forcing (force CLM) - CLM1PT = Run with supplied single point data (force CLM) - CORE2_NYF = CORE2 normal year forcing (for forcing POP and CICE) - CORE2_IAF = CORE2 intra-annual year forcing (for forcing POP and CICE) - CPLHIST3HrWx = Streams for 3-hourly coupler history output (for forcing CLM) - CPLHISTForcingForOcnIce = Streams for ocn/ice forcing used for spinup (for forcing POP and CICE) - presaero = Prescribed aerosol forcing - topo = Surface topography - co2tseries = Time series of prescribed CO2 forcing - Anomaly.Forcing = Time series of correction terms - BC = Bias Correction for a stream to a given set of observations - WW3 = Wave model forcing + CLM_QIAN = Run with the CLM Qian dataset valid from 1948 to 2004 (force CLM) + CLM_QIAN_WISO = Run with the CLM Qian dataset with isotopes valid from 2000 to 2004 (force CLM) + CLMCRUNCEP = Run with the CLM CRU NCEP V4 ( default ) forcing valid from 1900 to 2010 (force CLM) + CLMCRUNCEP_V5 = Run with the CLM CRU NCEP V5 forcing valid from 1900 to 2010 (force CLM) + CLMGSWP3 = Run with the CLM GSWP3 forcing (force CLM) + CLM1PT = Run with supplied single point data (force CLM) + CORE2_NYF = CORE2 normal year forcing (for forcing POP and CICE) + CORE2_IAF = CORE2 intra-annual year forcing (for forcing POP and CICE) + CPLHISTForcing = Streams for lnd or ocn/ice forcing used for spinup + presaero = Prescribed aerosol forcing + topo = Surface topography + co2tseries = Time series of prescribed CO2 forcing + Anomaly.Forcing = Time series of correction terms + BC = Bias Correction for a stream to a given set of observations Anomaly.Forcing.Humidity Anomaly.Forcing.Longwave @@ -73,14 +71,10 @@ CLM1PT.1x1_urbanc_alpha CLM1PT.CLM_USRDAT - CPLHIST3Hrly.Solar - CPLHIST3Hrly.Precip - CPLHIST3HrWx.nonSolarNonPrecip - - CPLHISTForcingForOcnIce.Solar - CPLHISTForcingForOcnIce.nonSolarFlux - CPLHISTForcingForOcnIce.State3hr - CPLHISTForcingForOcnIce.State1hr + CPLHISTForcing.Solar + CPLHISTForcing.nonSolarFlux + CPLHISTForcing.State3hr + CPLHISTForcing.State1hr CLM_QIAN.Solar CLM_QIAN.Precip @@ -113,9 +107,8 @@ CORE2_NYF.NCEP Note for CORE2_IAF: - The most current versions of forcing files - (those recommended for use) are duplicated - below and stored at /ccsm/ocn/iaf/): + The most current versions of forcing files(those recommended for use) are duplicated + below and stored at /ccsm/ocn/iaf/): gcgcs.prec.T62.current, giss.lwdn.T62.current, giss.swdn.T62.current, giss.swup.T62.current, ncep.dn10.T62.current, ncep.q_10.T62.current ncep.slp_.T62.current, ncep.t_10.T62.current, ncep.u_10.T62.current, ncep.v_10.T62.current @@ -143,8 +136,6 @@ topo.observed - WW3 - **********IMPORTANT NOTE: ************* In the value matches below, regular expressions are used **** If two matches are equivalent, the FIRST one will be used, so need to make sure @@ -180,10 +171,7 @@ CORE2_NYF.GISS,CORE2_NYF.GXGXS,CORE2_NYF.NCEP CORE2_IAF.GCGCS.PREC,CORE2_IAF.GISS.LWDN,CORE2_IAF.GISS.SWDN,CORE2_IAF.GISS.SWUP,CORE2_IAF.NCEP.DN10,CORE2_IAF.NCEP.Q_10,CORE2_IAF.NCEP.SLP_,CORE2_IAF.NCEP.T_10,CORE2_IAF.NCEP.U_10,CORE2_IAF.NCEP.V_10,CORE2_IAF.CORE2.ArcFactor CORE2_IAF.NCEP.DENS.SOFS,CORE2_IAF.NCEP.PSLV.SOFS,CORE2_IAF.PREC.SOFS.DAILY,CORE2_IAF.LWDN.SOFS.DAILY,CORE2_IAF.SWDN.SOFS.DAILY,CORE2_IAF.SWUP.SOFS.DAILY,CORE2_IAF.SHUM.SOFS.6HOUR,CORE2_IAF.TBOT.SOFS.6HOUR,CORE2_IAF.U.SOFS.6HOUR,CORE2_IAF.V.SOFS.6HOUR,CORE2_IAF.CORE2.ArcFactor - WW3 - NLDAS - CPLHIST3HrWx.Solar,CPLHIST3HrWx.Precip,CPLHIST3HrWx.nonSolarNonPrecip - CPLHISTForcingForOcnIce.Solar,CPLHISTForcingForOcnIce.nonSolarFlux,CPLHISTForcingForOcnIce.State3hr,CPLHISTForcingForOcnIce.State1hr + CPLHISTForcing.Solar,CPLHISTForcing.nonSolarFlux,CPLHISTForcing.State3hr,CPLHISTForcing.State1hr @@ -196,8 +184,7 @@ null $DIN_LOC_ROOT/atm/datm7/domain.clm $ATM_DOMAIN_PATH - $DIN_LOC_ROOT/atm/datm7 - $DATM_CPLHIST_DIR + $DATM_CPLHIST_DIR $DIN_LOC_ROOT/atm/datm7 $DIN_LOC_ROOT/share/domains/domain.clm $DIN_LOC_ROOT_CLMFORC/atm_forcing.datm7.cruncep_qianFill.0.5d.V5.c140715 @@ -224,8 +211,6 @@ $DIN_LOC_ROOT/atm/datm7 $DIN_LOC_ROOT/atm/datm7 $DIN_LOC_ROOT/atm/datm7 - $DIN_LOC_ROOT/wav/ww3 - $DIN_LOC_ROOT/share/domains/domain.clm $DIN_LOC_ROOT/atm/datm7/CO2 $DIN_LOC_ROOT/atm/datm7/bias_correction/precip/gpcp/qian $DIN_LOC_ROOT/atm/datm7/clm_output/cruncep_precip_1deg/gpcp_1deg_bias_correction @@ -251,8 +236,7 @@ domain.lnd.1x1pt-vancouverCAN_navy.090715.nc domain.lnd.1x1pt-urbanc_alpha_navy.090715.nc $ATM_DOMAIN_FILE - domain.lnd.fv0.9x1.25_gx1v6.090309.nc - null + null domain.T62.050609.nc domain.T62.050609.nc domain.cruncep.V5.c2013.0.5d.nc @@ -282,8 +266,6 @@ domain.T62.050609.nc domain.T62.050609.nc CORE2.t_10.ArcFactor.T62.1997-2004.nc - core2_G4_wns_30min_20000601_to_05.nc - domain.lnd.nldas2_0224x0464_c110415.nc fco2_datm_1765-2007_c100614.nc fco2_datm_rcp2.6_1765-2500_c130312.nc fco2_datm_rcp4.5_1765-2500_c130312.nc @@ -328,7 +310,7 @@ area area mask mask - + time time doma_lon lon doma_lat lat @@ -349,13 +331,6 @@ area area mask mask - - time time - gridLon lon - gridLat lat - area area - mask mask - time time LONGXY lon @@ -384,8 +359,7 @@ $DIN_LOC_ROOT/atm/datm7/CLM1PT_data/vancouverCAN.c080124 $DIN_LOC_ROOT/atm/datm7/CLM1PT_data/urbanc_alpha.c080416 $DIN_LOC_ROOT_CLMFORC/$CLM_USRDAT_NAME/CLM1PT_data - /glade/p/cesm/shared_outputdata/cases/ccsm4/$DATM_CPLHIST_CASE/cpl/hist - $DATM_CPLHIST_DIR + $DATM_CPLHIST_DIR $DIN_LOC_ROOT/atm/datm7/atm_forcing.datm7.Qian.T62.c080727/Solar6Hrly $DIN_LOC_ROOT/atm/datm7/atm_forcing.datm7.Qian.T62.c080727/Precip6Hrly $DIN_LOC_ROOT/atm/datm7/atm_forcing.datm7.Qian.T62.c080727/TmpPrsHumWnd3Hrly @@ -404,8 +378,6 @@ $DIN_LOC_ROOT/atm/datm7/NYF $DIN_LOC_ROOT/atm/datm7/CORE2 $DIN_LOC_ROOT/ocn/iaf - $DIN_LOC_ROOT/wav/ww3 - $DIN_LOC_ROOT/atm/datm7/NLDAS $DIN_LOC_ROOT/atm/datm7/CO2 $DIN_LOC_ROOT/atm/datm7/bias_correction/precip/gpcp/qian $DIN_LOC_ROOT/atm/datm7/clm_output/cruncep_precip_1deg/gpcp_1deg_bias_correction @@ -445,13 +417,10 @@ clm1pt-0002-11.nc %ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x1hi.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc - $DATM_CPLHIST_CASE.cpl.ha2x1h.%ym.nc + $DATM_CPLHIST_CASE.cpl.ha2x1hi.%ym.nc + $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc + $DATM_CPLHIST_CASE.cpl.ha2x3h.%ym.nc + $DATM_CPLHIST_CASE.cpl.ha2x1h.%ym.nc $DATM_CPLHIST_CASE.cpl.ha2x1d.%ym.nc clmforc.Qian.c2006.T62.Solr.%ym.nc clmforc.Qian.c2006.T62.Prec.%ym.nc @@ -1122,21 +1091,6 @@ ncep.v_10.T62.2009.20120412.nc CORE2.t_10.ArcFactor.T62.1997-2004.nc - core2_G4_wns_30min_20000601_to_05.nc - - clmforc.nldas.1980-01.nc - clmforc.nldas.1980-02.nc - clmforc.nldas.1980-03.nc - clmforc.nldas.1980-04.nc - clmforc.nldas.1980-05.nc - clmforc.nldas.1980-06.nc - clmforc.nldas.1980-07.nc - clmforc.nldas.1980-08.nc - clmforc.nldas.1980-09.nc - clmforc.nldas.1980-10.nc - clmforc.nldas.1980-11.nc - clmforc.nldas.1980-12.nc - fco2_datm_1765-2007_c100614.nc fco2_datm_rcp2.6_1765-2500_c130312.nc fco2_datm_rcp4.5_1765-2500_c130312.nc @@ -1226,44 +1180,20 @@ PSRF pbot FLDS lwdn - - a2x3h_Faxa_swndr swndr - a2x3h_Faxa_swvdr swvdr - a2x3h_Faxa_swndf swndf - a2x3h_Faxa_swvdf swvdf - - - a2x3h_Faxa_rainc rainc - a2x3h_Faxa_rainl rainl - a2x3h_Faxa_snowc snowc - a2x3h_Faxa_snowl snowl - - - a2x3h_Sa_z z - a2x3h_Sa_u u - a2x3h_Sa_v v - a2x3h_Sa_tbot tbot - a2x3h_Sa_ptem ptem - a2x3h_Sa_shum shum - a2x3h_Sa_pbot pbot - a2x3h_Faxa_lwdn lwdn - a2x3h_Sa_dens dens - a2x3h_Sa_pslv pslv - - + a2x1hi_Faxa_swndr swndr a2x1hi_Faxa_swvdr swvdr a2x1hi_Faxa_swndf swndf a2x1hi_Faxa_swvdf swvdf - + a2x3h_Faxa_rainc rainc a2x3h_Faxa_rainl rainl a2x3h_Faxa_snowc snowc a2x3h_Faxa_snowl snowl a2x3h_Faxa_lwdn lwdn - + a2x3h_Sa_z z a2x3h_Sa_tbot tbot a2x3h_Sa_ptem ptem @@ -1274,7 +1204,7 @@ a2x3h_Sa_co2diag co2diag a2x3h_Sa_co2prog co2prog - + a2x1h_Sa_u u a2x1h_Sa_v v @@ -1418,20 +1348,6 @@ TarcFactor tarcf - - uWind u - vWind v - airSeaTempDiff tbot - - - TBOT tbot - WIND wind - QBOT shum - PSRF pbot - FLDS lwdn - PRECTmms precn - FSDS swdn - CO2 co2diag @@ -1526,16 +1442,13 @@ -999 $DATM_CLMNCEP_YR_ALIGN - $DATM_CPLHIST_YR_ALIGN - $DATM_CPLHIST_YR_ALIGN + $DATM_CPLHIST_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN $DATM_CLMNCEP_YR_ALIGN 1 1 1 - 2000 - 1980 1850 2005 1979 @@ -1561,8 +1474,7 @@ 1992 0001 $DATM_CLMNCEP_YR_START - $DATM_CPLHIST_YR_START - $DATM_CPLHIST_YR_START + $DATM_CPLHIST_YR_START 2000 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START @@ -1589,8 +1501,6 @@ 1948 1948 1948 - 2000 - 1980 1850 2005 1979 @@ -1615,8 +1525,7 @@ 1992 0002 $DATM_CLMNCEP_YR_END - $DATM_CPLHIST_YR_END - $DATM_CPLHIST_YR_END + $DATM_CPLHIST_YR_END 2004 $DATM_CLMNCEP_YR_END $DATM_CLMNCEP_YR_END @@ -1643,8 +1552,6 @@ 2009 2009 2009 - 2000 - 1980 2007 2500 2004 @@ -1666,13 +1573,10 @@ Stream offset. 0 - -10800 - -5400 - -5400 - 2700 - 900 - 900 - 900 + 2700 + 900 + 900 + 900 0 @@ -1738,12 +1642,9 @@ NULL CLMNCEP - CPLHIST CORE2_NYF CORE2_IAF - COPYALL - CLMNCEP - CPLHIST + CPLHIST @@ -1912,12 +1813,10 @@ nearest nearest nearest - coszen - nearest - nearest - nearest - linear - linear + nearest + nearest + linear + linear nearest coszen nearest @@ -1992,10 +1891,10 @@ 1.5 1.e30 - 2.0 - 2.0 - 2.0 - 2.0 + 2.0 + 2.0 + 2.0 + 2.0 2.0 @@ -2015,12 +1914,8 @@ null - null - u:v u:v u:v - u:v - null @@ -2148,7 +2043,7 @@ 1 - -1 + -1 diff --git a/src/components/data_comps/datm/datm_comp_mod.F90 b/src/components/data_comps/datm/datm_comp_mod.F90 index 48a7d63e4171..e6099e42b63e 100644 --- a/src/components/data_comps/datm/datm_comp_mod.F90 +++ b/src/components/data_comps/datm/datm_comp_mod.F90 @@ -475,7 +475,6 @@ subroutine datm_comp_init( EClock, cdata, x2a, a2x, NLFilename ) if (trim(atm_mode) == 'NULL' .or. & trim(atm_mode) == 'CORE2_NYF' .or. & trim(atm_mode) == 'CORE2_IAF' .or. & - trim(atm_mode) == 'WRF' .or. & trim(atm_mode) == 'CLMNCEP' .or. & trim(atm_mode) == 'CPLHIST' .or. & trim(atm_mode) == 'COPYALL' ) then @@ -975,31 +974,6 @@ subroutine datm_comp_run( EClock, cdata, x2a, a2x) case('CPLHIST') ! do nothing extra - case ('WRF') - lsize = mct_avect_lsize(a2x) - do n = 1,lsize - - !--- fabricate required swdn components from total swdn --- - a2x%rAttr(kswvdr,n) = avstrm%rAttr(sswdn,n)*(0.28_R8) - a2x%rAttr(kswndr,n) = avstrm%rAttr(sswdn,n)*(0.31_R8) - a2x%rAttr(kswvdf,n) = avstrm%rAttr(sswdn,n)*(0.24_R8) - a2x%rAttr(kswndf,n) = avstrm%rAttr(sswdn,n)*(0.17_R8) - - !--- just a diagnostic, not really needed - a2x%rAttr(kswnet,n) = avstrm%rAttr(sswdn,n)-avstrm%rAttr(sswup,n) - - !--- convert from hPa - a2x%rAttr(kpslv,n) = a2x%rAttr(kpslv,n)*100._R8 - a2x%rAttr(kpbot,n) = a2x%rAttr(kpbot,n)*100._R8 - - !--- tcraig, file has terrain height on it, set to 10m - a2x%rAttr(kz,n) = 10.0_R8 - - !--- convert to degK from degC - a2x%rAttr(ktbot,n) = a2x%rAttr(ktbot,n) + tKFrz - - enddo - case('CORE2_NYF','CORE2_IAF') if (firstcall) then if (sprec < 1 .or. sswdn < 1) then diff --git a/src/components/data_comps/desp/desp_comp_mod.F90 b/src/components/data_comps/desp/desp_comp_mod.F90 index ab86eb3c2419..d48b5e06a5d9 100644 --- a/src/components/data_comps/desp/desp_comp_mod.F90 +++ b/src/components/data_comps/desp/desp_comp_mod.F90 @@ -15,7 +15,8 @@ module desp_comp_mod use shr_strdata_mod, only: shr_strdata_type, shr_strdata_advance use shr_strdata_mod, only: shr_strdata_pioinit - use seq_timemgr_mod, only: seq_timemgr_EClockGetData, seq_timemgr_RestartAlarmIsOn + use seq_timemgr_mod, only: seq_timemgr_EClockGetData + use seq_timemgr_mod, only: seq_timemgr_RestartAlarmIsOn use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix implicit none @@ -30,39 +31,29 @@ module desp_comp_mod public :: desp_comp_final !-------------------------------------------------------------------------- -! Public data +! Public module data !-------------------------------------------------------------------------- - - integer, parameter, public :: atm_ind = 1 - integer, parameter, public :: lnd_ind = 2 - integer, parameter, public :: ice_ind = 3 - integer, parameter, public :: ocn_ind = 4 - integer, parameter, public :: glc_ind = 5 - integer, parameter, public :: rof_ind = 6 - integer, parameter, public :: wav_ind = 7 - integer, parameter, public :: cpl_ind = 8 - integer, parameter, public :: max_ind = cpl_ind + integer, public, parameter :: desp_num_comps = 8 + character(len=3), public, parameter :: comp_names(desp_num_comps) = & + (/ 'atm', 'lnd', 'ice', 'ocn', 'glc', 'rof', 'wav', 'drv' /) !-------------------------------------------------------------------------- ! Private data !-------------------------------------------------------------------------- - character(len=CS) :: myModelName = 'esp' ! user defined model name + character(len=CS) :: myModelName = 'esp' ! user defined model name integer(IN) :: mpicom - integer(IN) :: COMPID ! mct comp id - integer(IN) :: my_task ! my task in mpi communicator mpicom - integer(IN) :: npes ! total number of tasks - integer(IN), parameter :: master_task=0 ! task number of master task - integer(IN) :: logunit ! logging unit number + integer(IN) :: COMPID ! mct comp id + integer(IN) :: my_task ! my task in mpi communicator mpicom + integer(IN) :: npes ! total number of tasks + integer(IN), parameter :: master_task=0 ! task number of master task + integer(IN) :: logunit ! logging unit number integer(IN) :: loglevel - integer :: inst_index ! number of current instance (ie. 1) - character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") - character(len=16) :: inst_suffix ! char string associated with instance - ! (ie. "_0001" or "") - character(len=CL) :: desp_mode ! mode of operation - - character(len=3), parameter :: comp_names(8) = (/ 'atm', 'lnd', 'ice', & - 'ocn', 'glc', 'rof', 'wav', 'drv' /) + integer :: inst_index ! number of current instance (ie. 1) + character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") + character(len=16) :: inst_suffix ! char string associated with instance + ! (ie. "_0001" or "") + character(len=CL) :: desp_mode ! mode of operation character(len=*), parameter :: rpprefix = 'rpointer.' character(len=*), parameter :: rpfile = rpprefix//'esp' character(len=*), parameter :: nullstr = 'undefined' @@ -148,16 +139,10 @@ subroutine desp_comp_init(EClock, espid, mpicom_in, phase, read_restart, & desp_mode, info_debug, restfilm !--- formats --- - character(*), parameter :: F00 = "('(desp_comp_init) ',8a)" - character(*), parameter :: F0L = "('(desp_comp_init) ',a, l2)" - character(*), parameter :: F01 = "('(desp_comp_init) ',a,5i8)" - character(*), parameter :: F02 = "('(desp_comp_init) ',a,4es13.6)" - character(*), parameter :: F03 = "('(desp_comp_init) ',a,i8,a)" - character(*), parameter :: F04 = "('(desp_comp_init) ',2a,2i8,'s')" - character(*), parameter :: F05 = "('(desp_comp_init) ',a,2f10.4)" - character(*), parameter :: F90 = "('(desp_comp_init) ',73('='))" - character(*), parameter :: F91 = "('(desp_comp_init) ',73('-'))" character(*), parameter :: subName = "(desp_comp_init) " + character(*), parameter :: F00 = "('"//subName//"',8a)" + character(*), parameter :: F01 = "('"//subName//"',a,5i8)" + character(*), parameter :: F04 = "('"//subName//"',2a,2i8,'s')" !------------------------------------------------------------------------------- call t_startf('DESP_INIT') @@ -219,8 +204,8 @@ subroutine desp_comp_init(EClock, espid, mpicom_in, phase, read_restart, & write(logunit,F01) 'ERROR: reading input namelist, '//trim(filename)//' iostat=',ierr call shr_sys_abort(subName//': namelist read error '//trim(filename)) end if - write(logunit,F01)' info_debug = ',info_debug - write(logunit,F00)' restfilm = ',trim(restfilm) + write(logunit,F01) 'info_debug = ',info_debug + write(logunit,F00) 'restfilm = ',trim(restfilm) write(logunit,F01) 'inst_index = ',inst_index write(logunit,F00) 'inst_name = ',trim(inst_name) write(logunit,F00) 'inst_suffix = ',trim(inst_suffix) @@ -360,7 +345,7 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & type(ESMF_Clock), intent(in) :: EClock character(len=*), intent(in) :: case_name - logical, intent(in) :: pause_sig(max_ind) + logical, intent(in) :: pause_sig(desp_num_comps) character(len=CL), intent(inout) :: atm_resume(num_inst_atm) character(len=CL), intent(inout) :: lnd_resume(num_inst_lnd) character(len=CL), intent(inout) :: rof_resume(num_inst_rof) @@ -445,45 +430,45 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & ! Find the active components and their restart files ! Note hard-coded variable names are just for testing. This could be ! changed if this feature comes to regular use - do ind = 1, max_ind + do ind = 1, desp_num_comps if (pause_sig(ind)) then - select case (ind) - case(atm_ind) + select case (comp_names(ind)) + case('atm') call get_restart_filenames(ind, atm_resume, errcode) allocate(rfilenames(size(atm_resume))) rfilenames = atm_resume varname = 'T' - case(lnd_ind) + case('lnd') call get_restart_filenames(ind, lnd_resume, errcode) allocate(rfilenames(size(lnd_resume))) rfilenames = lnd_resume varname = 'T' - case(ice_ind) + case('ice') call get_restart_filenames(ind, ice_resume, errcode) allocate(rfilenames(size(ice_resume))) rfilenames = ice_resume varname = 'T' - case(ocn_ind) + case('ocn') call get_restart_filenames(ind, ocn_resume, errcode) allocate(rfilenames(size(ocn_resume))) rfilenames = ocn_resume - varname = 'T' - case(glc_ind) + varname = 'PSURF_CUR' + case('glc') call get_restart_filenames(ind, glc_resume, errcode) allocate(rfilenames(size(glc_resume))) rfilenames = glc_resume varname = 'T' - case(rof_ind) + case('rof') call get_restart_filenames(ind, rof_resume, errcode) allocate(rfilenames(size(rof_resume))) rfilenames = rof_resume varname = 'T' - case(wav_ind) + case('wav') call get_restart_filenames(ind, wav_resume, errcode) allocate(rfilenames(size(wav_resume))) rfilenames = wav_resume varname = 'T' - case(cpl_ind) + case('drv') call get_restart_filenames(ind, cpl_resume, errcode) allocate(rfilenames(1)) rfilenames(1) = cpl_resume @@ -520,6 +505,9 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & write(logunit, *) subname, 'Found restart file ',trim(rfilenames(inst)) end if call esp_pio_modify_variable(COMPID, mpicom, rfilenames(inst), varname, var_found) + if (.not. var_found) then + call shr_sys_abort(subname//'Variable, '//trim(varname)//', not found on '//rfilenames(inst)) + end if end do case (null_mode) ! Since DESP is not 'present' for this mode, we should not get here. @@ -642,22 +630,22 @@ subroutine get_restart_filenames_a(comp_ind, filenames, retcode) filenames = ' ' num_inst = size(filenames) allocate(ids(num_inst)) - select case (comp_ind) - case(atm_ind) + select case (comp_names(comp_ind)) + case('atm') ids = ATMID - case(lnd_ind) + case('lnd') ids = LNDID - case(ice_ind) + case('ice') ids = ICEID - case(ocn_ind) + case('ocn') ids = OCNID - case(glc_ind) + case('glc') ids = GLCID - case(rof_ind) + case('rof') ids = ROFID - case(wav_ind) + case('wav') ids = WAVID - case(cpl_ind) + case('drv') ids = CPLID case default call shr_sys_abort(subname//'Unrecognized comp_ind') @@ -668,6 +656,11 @@ subroutine get_restart_filenames_a(comp_ind, filenames, retcode) rpointer_name = rpprefix//comp_names(comp_ind)//trim(seq_comm_suffix(ids(ind))) if (my_task == master_task) then inquire(file=rpointer_name, EXIST=file_exists) + ! POP decided to not follow the convention + if ((.not. file_exists) .and. (trim(comp_names(comp_ind)) == 'ocn')) then + rpointer_name = rpprefix//comp_names(comp_ind)//".restart"//trim(seq_comm_suffix(ids(ind))) + inquire(file=rpointer_name, EXIST=file_exists) + end if if (.not. file_exists) then retcode = NO_RPOINTER else diff --git a/src/components/data_comps/desp/esp_comp_mct.F90 b/src/components/data_comps/desp/esp_comp_mct.F90 index 549a1ce1027c..3f3c07a98874 100644 --- a/src/components/data_comps/desp/esp_comp_mct.F90 +++ b/src/components/data_comps/desp/esp_comp_mct.F90 @@ -2,10 +2,11 @@ module esp_comp_mct ! !USES: - use esmf, only: ESMF_Clock - use mct_mod, only: mct_aVect - use seq_cdata_mod, only: seq_cdata - use seq_infodata_mod, only: seq_infodata_type + use esmf, only: ESMF_Clock + use mct_mod, only: mct_aVect + use seq_cdata_mod, only: seq_cdata + use seq_infodata_mod, only: seq_infodata_type + use desp_comp_mod, only: desp_num_comps ! !PUBLIC TYPES: implicit none @@ -19,6 +20,11 @@ module esp_comp_mct public :: esp_run_mct public :: esp_final_mct + !-------------------------------------------------------------------------- + ! Private module data + !-------------------------------------------------------------------------- + integer :: comp_index(desp_num_comps) + !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -36,9 +42,11 @@ module esp_comp_mct ! !INTERFACE: --------------------------------------------------------------- subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) - use desp_comp_mod, only: desp_comp_init + use desp_comp_mod, only: desp_comp_init, comp_names use seq_cdata_mod, only: seq_cdata_setptrs use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData + use seq_timemgr_mod, only: seq_timemgr_pause_component_index + ! !INPUT/OUTPUT PARAMETERS: @@ -49,6 +57,7 @@ subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) !EOP + integer :: ind integer :: ESPID integer :: mpicom_esp integer :: esp_phase @@ -73,6 +82,11 @@ subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) call seq_infodata_PutData(infodata, & esp_present=esp_present, esp_prognostic=esp_prognostic) + ! Retrieve component indices from the time manager + do ind = 1, desp_num_comps + comp_index(ind) = seq_timemgr_pause_component_index(comp_names(ind)) + end do + end subroutine esp_init_mct !============================================================================ @@ -88,14 +102,14 @@ end subroutine esp_init_mct ! !INTERFACE: ------------------------------------------------------------------ subroutine esp_run_mct( EClock, cdata, x2a, a2x) + use shr_kind_mod, only: CL=>SHR_KIND_CL use seq_comm_mct, only: num_inst_atm, num_inst_lnd, num_inst_rof use seq_comm_mct, only: num_inst_ocn, num_inst_ice, num_inst_glc use seq_comm_mct, only: num_inst_wav - use desp_comp_mod, only: desp_comp_run, atm_ind, lnd_ind, ocn_ind - use desp_comp_mod, only: ice_ind, glc_ind, rof_ind, wav_ind, cpl_ind, max_ind + use desp_comp_mod, only: desp_comp_run use seq_cdata_mod, only: seq_cdata_setptrs use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData - use shr_kind_mod, only: CL=>SHR_KIND_CL + use seq_timemgr_mod, only: seq_timemgr_pause_component_active ! !INPUT/OUTPUT PARAMETERS: @@ -106,8 +120,9 @@ subroutine esp_run_mct( EClock, cdata, x2a, a2x) !EOP + integer :: ind type(seq_infodata_type), pointer :: infodata - logical :: pause_sig(max_ind) + logical :: pause_sig(desp_num_comps) character(len=CL) :: atm_resume(num_inst_atm) character(len=CL) :: lnd_resume(num_inst_lnd) character(len=CL) :: rof_resume(num_inst_rof) @@ -120,16 +135,13 @@ subroutine esp_run_mct( EClock, cdata, x2a, a2x) character(len=*), parameter :: subName = "(esp_run_mct) " !-------------------------------------------------------------------------- - ! Grab infodata + ! Grab infodata and case name call seq_cdata_setptrs(cdata, infodata=infodata) + call seq_infodata_GetData(infodata, case_name=case_name) ! Find out if we should be running - ! The data ESP component only runs during a pause alarm - call seq_infodata_GetData(infodata, atm_pause=pause_sig(atm_ind), & - lnd_pause=pause_sig(lnd_ind), ocn_pause=pause_sig(ocn_ind), & - ice_pause=pause_sig(ice_ind), glc_pause=pause_sig(glc_ind), & - rof_pause=pause_sig(rof_ind), wav_pause=pause_sig(wav_ind), & - cpl_pause=pause_sig(cpl_ind), case_name=case_name) - + do ind = 1, desp_num_comps + pause_sig(ind) = seq_timemgr_pause_component_active(comp_index(ind)) + end do call desp_comp_run(EClock, case_name, pause_sig, atm_resume, lnd_resume, & rof_resume, ocn_resume, ice_resume, glc_resume, wav_resume, & cpl_resume) diff --git a/src/components/data_comps/dice/cime_config/config_component.xml b/src/components/data_comps/dice/cime_config/config_component.xml index 26591c92f2bb..92bc2a30261c 100644 --- a/src/components/data_comps/dice/cime_config/config_component.xml +++ b/src/components/data_comps/dice/cime_config/config_component.xml @@ -21,7 +21,6 @@ ssmi ssmi_iaf prescribed - copyall null run_component_dice @@ -48,7 +47,6 @@ dice mode is ssmi: dice mode is ssmi_iaf: dice mode is prescribed: - dice mode is copy: dice mode is null: diff --git a/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml b/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml index 57b32f6bce56..e5b2cdb7db04 100644 --- a/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml +++ b/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml @@ -25,9 +25,9 @@ 'streamN.txt year_align year_first year_last ' Currently the following datamode values are supported - SSMI_IAF - SSMI NULL + COPYALL + SSTDATA **********IMPORTANT NOTE: ************* In the value matches below, regular expressions are used **** @@ -55,11 +55,10 @@ streams_file List of streams used for the given dice_mode. - NULL - SSMI - SSMI_IAF + NULL + SSMI + SSMI_IAF prescribed - copyall @@ -69,7 +68,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -80,7 +78,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -122,11 +119,6 @@ area area mask mask - - time time - gridLon lon - gridLat lat - @@ -226,9 +218,6 @@ ifrac ifrac - - iceCon ifrac - @@ -252,7 +241,6 @@ 1 1 $SSTICE_YEAR_ALIGN - $SSTICE_YEAR_ALIGN @@ -266,7 +254,6 @@ 1 1948 $SSTICE_YEAR_START - $SSTICE_YEAR_START @@ -280,7 +267,6 @@ 1 2009 $SSTICE_YEAR_END - $SSTICE_YEAR_END @@ -326,11 +312,10 @@ If DICE_MODE is set to null, datamodel will be set to NULL - NULL - SSTDATA - SSTDATA + NULL + SSTDATA + SSTDATA SSTDATA - COPYALL @@ -347,7 +332,7 @@ $ICE_DOMAIN_PATH/$ICE_DOMAIN_FILE $SSTICE_GRID_FILENAME - $SSTICE_GRID_FILENAME + $SSTICE_GRID_FILENAME diff --git a/src/components/data_comps/dice/dice_comp_mod.F90 b/src/components/data_comps/dice/dice_comp_mod.F90 index ec31b04cdb7f..692b07c2c645 100644 --- a/src/components/data_comps/dice/dice_comp_mod.F90 +++ b/src/components/data_comps/dice/dice_comp_mod.F90 @@ -621,7 +621,6 @@ subroutine dice_comp_run( EClock, cdata, x2i, i2x) integer(IN) :: nl ! ice frac index integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level - logical :: glcrun_alarm ! is glc going to run now logical :: newdata ! has newdata been read logical :: mssrmlf ! remove old data integer(IN) :: idt ! integer timestep diff --git a/src/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml index 09d88806eb7e..46b0d82ad1a0 100755 --- a/src/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -1,6 +1,6 @@ #!/usr/bin/env python -"""Namelist creator for CIME's data ocn model. +"""Namelist creator for CIME's data land model. """ # Typically ignore this. @@ -106,6 +106,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): nmlgen.update_shr_strdata_nml(config, stream, stream_path) else: nmlgen.create_stream_file_and_update_shr_strdata_nml(config, stream, stream_path, data_list_path) + #---------------------------------------------------- # Create `shr_strdata_nml` namelist group. #---------------------------------------------------- diff --git a/src/components/data_comps/dlnd/cime_config/config_component.xml b/src/components/data_comps/dlnd/cime_config/config_component.xml index f74f63c47d79..3e8d2a02d615 100644 --- a/src/components/data_comps/dlnd/cime_config/config_component.xml +++ b/src/components/data_comps/dlnd/cime_config/config_component.xml @@ -34,11 +34,11 @@ char UNSET - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG1850CN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BGRCP85CN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG1850CN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BGRCP85CN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 run_component_dlnd env_run.xml @@ -49,11 +49,11 @@ char UNSET - b.e10.BG20TRCN.f09_g16.002 - b.e10.BG1850CN.f09_g16.002 - b.e10.BG20TRCN.f09_g16.002 - b.e10.BG20TRCN.f09_g16.002 - b.e10.BGRCP85CN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BG1850CN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BGRCP85CN.f09_g16.002 run_component_dlnd env_run.xml @@ -64,11 +64,11 @@ integer 1 - 1 - 1 - 1850 - 1850 - 2006 + 1 + 1 + 1850 + 1850 + 2006 run_component_dlnd env_run.xml @@ -79,11 +79,11 @@ integer 1 - 1976 - 26 - 1850 - 1850 - 2006 + 1976 + 26 + 1850 + 1850 + 2006 run_component_dlnd env_run.xml @@ -94,11 +94,11 @@ integer 1 - 2005 - 100 - 2005 - 2005 - 2100 + 2005 + 100 + 2005 + 2005 + 2100 run_component_dlnd env_run.xml diff --git a/src/components/data_comps/docn/cime_config/buildnml b/src/components/data_comps/docn/cime_config/buildnml index 2afb78283e54..360c84e2db27 100755 --- a/src/components/data_comps/docn/cime_config/buildnml +++ b/src/components/data_comps/docn/cime_config/buildnml @@ -9,7 +9,7 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os, shutil, sys, glob +import os, shutil, sys, glob, re _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) @@ -108,6 +108,13 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): else: nmlgen.create_stream_file_and_update_shr_strdata_nml(config, stream, stream_path, data_list_path) + # For aquaplanet prescribed have no streams + match = re.match(r'^sst_aquap\d+',docn_mode) + if match is not None: + if match.group(0): + value = ['null'] + nmlgen.set_value("streams",value) + #---------------------------------------------------- # Create `shr_strdata_nml` namelist group. #---------------------------------------------------- diff --git a/src/components/data_comps/docn/cime_config/config_component.xml b/src/components/data_comps/docn/cime_config/config_component.xml index 165d21c1dadb..ff71aa17f1b2 100644 --- a/src/components/data_comps/docn/cime_config/config_component.xml +++ b/src/components/data_comps/docn/cime_config/config_component.xml @@ -15,14 +15,24 @@ char - prescribed,som,copyall,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,som,som_aquap,copyall,interannual,null prescribed null prescribed som - us20 + som_aquap interannual + sst_aquap1 + sst_aquap2 + sst_aquap3 + sst_aquap4 + sst_aquap5 + sst_aquap6 + sst_aquap7 + sst_aquap8 + sst_aquap9 + sst_aquap10 copyall run_component_docn @@ -31,7 +41,7 @@ driver. The atmosphere/ocean fluxes are computed in the coupler. Therefore, the data ocean model does not compute fluxes like the data ice model. DOCN has two distinct modes of operation. It can - arun as a pure data model, reading in ocean SSTs (normally + run as a pure data model, reading in ocean SSTs (normally climatological) from input datasets, performing time/spatial interpolations, and passing these to the coupler. Alternatively, DOCN can compute updated SSTs by running as a slab ocean model where @@ -40,18 +50,14 @@ from the driver. --- A setting of prescribed assumes the only field in the input stream is SST. It also assumes the SST is in Celsius and must be converted to Kelvin. - All other fields are set to zero except for ocean salinity, which - is set to a constant reference salinity value. - Normally the ice fraction data is found in the same data files that - provide SST data to the data ocean model. They are normally found in - the same file because the SST and ice fraction data are derived from - the same observational data sets and are consistent with each other. - to the data ocean model. They are normally found in the same file - because the SST and ice fraction data are derived from the same - observational data sets and are consistent with each other. - --- A setting of som (slab ocean model) mode is a prognostic mode. This mode - computes a prognostic sea surface temperature and a freeze/melt - potential (surface Q-flux) used by the sea ice model. This + All other fields are set to zero except for ocean salinity, which is set to a + constant reference salinity value. Normally the ice fraction data is found in + the same data files that provide SST data to the data ocean model. They are + normally found in the same file because the SST and ice fraction data are derived + from the same observational data sets and are consistent with each other. + --- Settings of som (slab ocean model) or som_aquap (aquaplanet slab ocean) are + prognostic modes which compute a prognostic sea surface temperature and a + freeze/melt potential (surface Q-flux) used by the sea ice model. This calculation requires an external SOM forcing data file that includes ocean mixed layer depths and bottom-of-the-slab Q-fluxes. Scientifically appropriate bottom-of-the-slab Q-fluxes are normally @@ -61,7 +67,14 @@ appropriate and is provided for testing and development purposes only. Users must create scientifically appropriate data for their particular application. A tool is available to derive valid SOM forcing. - Default is prescribed. + --- A setting of sst_aquapN (where “N” is an integer index value) is a + type of prescribed SST mode used specifically for an aquaplanet setup in + which global SSTs correspond to an analytic form set by the index value. + Currently, indices for 10 SST profiles are supported [e.g., index 3 corresponds + to the “QOBS” profile of Neale and Hoskins (2001, Atmos. Sci. Lett.)]. + With source code modifications, it is possible for users to create their own + analytic SST distributions and match them to indices 11 or greater. + @@ -69,7 +82,9 @@ UNSET - pop_frc.1x1d.090130.nc + pop_frc.1x1d.090130.nc + default.som.forcing.aquaplanet.Qflux0_h30_sstQOBS.2degFV_c20170421.nc + default.som.forcing.aquaplanet.Qflux0_h30_sstQOBS.1degFV_c20170421.nc run_component_docn env_run.xml @@ -80,9 +95,6 @@ char CAMDATA - - WW3 - run_component_docn env_run.xml Prescribed SST and ice coverage stream name. @@ -100,9 +112,9 @@ $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_clim_c061106.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_c110526.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_clim_pi_c101029.nc @@ -115,7 +127,6 @@ $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc - $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc run_component_docn env_run.xml @@ -152,7 +163,6 @@ $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc - $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc run_component_cam_sstice env_run.xml @@ -202,7 +212,6 @@ 0 2012 - 2000 run_component_cam_sstice env_run.xml @@ -213,12 +222,21 @@ - docn null mode: - docn slab ocean mode: - docn data mode: - docn us20 mode: - docn copy mode: - docn interannual mode: + docn null mode + docn slab ocean mode + docn aquaplanet slab ocean mode + docn interannual mode + docn aquaplanet mode: + docn prescribed aquaplanet sst - option 1 + docn prescribed aquaplanet sst - option 2 + docn prescribed aquaplanet sst - option 3 + docn prescribed aquaplanet sst - option 4 + docn prescribed aquaplanet sst - option 5 + docn prescribed aquaplanet sst - option 6 + docn prescribed aquaplanet sst - option 7 + docn prescribed aquaplanet sst - option 8 + docn prescribed aquaplanet sst - option 9 + docn prescribed aquaplanet sst - option 10 @@ -228,4 +246,3 @@ - diff --git a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 32d2d7f5ba5a..9f75abca4939 100644 --- a/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -33,7 +33,6 @@ prescribed SSTDATA (Run with prescribed SST, ICE_COV) som SOM (Slab ocean model) null NULL (NULL mode) - copyall COPYALL (Copy fields only) --> @@ -46,10 +45,20 @@ streams_file List of streams used for the given docn_mode. - prescribed - som + prescribed + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + som + som interannual - copyall @@ -59,7 +68,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -70,7 +78,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_DATA_FILENAME $SSTICE_DATA_FILENAME @@ -112,11 +119,6 @@ area area mask mask - - time time - gridLon lon - gridLat lat - time time lon lon @@ -172,9 +174,6 @@ sst t - - iceCon s - T t S s @@ -212,7 +211,6 @@ -999 $SSTICE_YEAR_ALIGN - $SSTICE_YEAR_ALIGN 1 1 @@ -226,7 +224,6 @@ -999 $SSTICE_YEAR_START - $SSTICE_YEAR_START 1 1850 @@ -239,10 +236,9 @@ Last year of stream. -999 - $SSTICE_YEAR_END - $SSTICE_YEAR_END - 1 - 2014 + $SSTICE_YEAR_END + 1 + 2014 @@ -258,7 +254,7 @@ char streams shr_strdata_nml - SSTDATA,SOM,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -305,16 +301,28 @@ If DOCN_MODE is prescribed, datamode will be set to SSTDATA If DOCN_MODE is interannual, datamode will be set to IAF If DOCN_MODE is som , datamode will be set to SOM + If DOCN_MODE is sst_aqup[n], datamode will be set to SST_AQUAP + If DOCN_MODE is som_aqup[n], datamode will be set to SOM_AQUAP If DOCN_MODE is null, datamode will be set to NULL default: SSTDATA (prescribed setting for DOCN_MODE)' - NULL - SSTDATA - SOM + NULL + SSTDATA + SST_AQUAP1 + SST_AQUAP2 + SST_AQUAP3 + SST_AQUAP4 + SST_AQUAP5 + SST_AQUAP6 + SST_AQUAP7 + SST_AQUAP8 + SST_AQUAP9 + SST_AQUAP10 + SOM + SOM_AQUAP IAF - COPYALL diff --git a/src/components/data_comps/docn/docn_comp_mod.F90 b/src/components/data_comps/docn/docn_comp_mod.F90 index 88cd3bc70a48..f557311f7f2b 100644 --- a/src/components/data_comps/docn/docn_comp_mod.F90 +++ b/src/components/data_comps/docn/docn_comp_mod.F90 @@ -79,13 +79,17 @@ module docn_comp_mod integer(IN) :: kt,ks,ku,kv,kdhdx,kdhdy,kq,kswp ! field indices integer(IN) :: kswnet,klwup,klwdn,ksen,klat,kmelth,ksnow,krofi integer(IN) :: kh,kqbot + integer(IN) :: index_lat, index_lon type(shr_strdata_type) :: SDOCN - type(mct_rearr) :: rearr - type(mct_avect) :: avstrm ! av of data from stream - real(R8), pointer :: somtp(:) - real(R8), pointer :: tfreeze(:) - integer , pointer :: imask(:) + type(mct_rearr) :: rearr + type(mct_avect) :: avstrm ! av of data from stream + real(R8), pointer :: somtp(:) + real(R8), pointer :: tfreeze(:) + integer , pointer :: imask(:) + real(R8), pointer :: xc(:), yc(:) ! arryas of model latitudes and longitudes + integer(IN) :: aquap_option + character(len=*),parameter :: flds_strm = 'strm_h:strm_qbot' integer(IN),parameter :: ktrans = 29 @@ -144,6 +148,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) integer(IN) :: shrlogunit, shrloglev ! original log unit and level integer(IN) :: nunit ! unit number integer(IN) :: kmask ! field reference + integer(IN) :: kfrac ! field reference logical :: ocn_present ! flag logical :: ocn_prognostic ! flag logical :: ocnrof_prognostic ! flag @@ -292,15 +297,31 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) ocn_mode = trim(SDOCN%dataMode) + ! Special logic for prescribed aquaplanet + if (ocn_mode(1:9) == 'SST_AQUAP') then + ! First determine the prescribed aquaplanet option + if (len_trim(ocn_mode) == 10) then + read(ocn_mode(10:10),'(i1)') aquap_option + else if (len_trim(ocn_mode) == 11) then + read(ocn_mode(10:11),'(i2)') aquap_option + end if + ! Now remove the index from the ocn_mode value, to have a generic setting + ! for use below + ocn_mode = "SST_AQUAP" + end if + ! check that we know how to handle the mode - if (trim(ocn_mode) == 'NULL' .or. & - trim(ocn_mode) == 'SSTDATA' .or. & - trim(ocn_mode) == 'COPYALL' .or. & - trim(ocn_mode) == 'IAF' .or. & - trim(ocn_mode) == 'SOM') then - if (my_task == master_task) & + if (trim(ocn_mode) == 'NULL' .or. & + trim(ocn_mode) == 'SSTDATA' .or. & + trim(ocn_mode) == 'SST_AQUAP' .or. & + trim(ocn_mode) == 'COPYALL' .or. & + trim(ocn_mode) == 'IAF' .or. & + trim(ocn_mode) == 'SOM' .or. & + trim(ocn_mode) == 'SOM_AQUAP') then + if (my_task == master_task) then write(logunit,F00) ' ocn mode = ',trim(ocn_mode) + end if else write(logunit,F00) ' ERROR illegal ocn mode = ',trim(ocn_mode) call shr_sys_abort() @@ -338,7 +359,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) ocnrof_prognostic = .true. endif - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then ocn_prognostic = .true. endif @@ -353,11 +374,11 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) !---------------------------------------------------------------------------- call seq_infodata_PutData(infodata, ocnrof_prognostic=ocnrof_prognostic, & - ocn_present=ocn_present, ocn_prognostic=ocn_prognostic, & - ocn_nx=SDOCN%nxg, ocn_ny=SDOCN%nyg ) + ocn_present=ocn_present, ocn_prognostic=ocn_prognostic, & + ocn_nx=SDOCN%nxg, ocn_ny=SDOCN%nyg ) !---------------------------------------------------------------------------- - ! Initialize MCT global seg map, 1d decomp + ! Initialize data model MCT global seg map, 1d decomp !---------------------------------------------------------------------------- call t_startf('docn_initgsmaps') @@ -374,14 +395,26 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) call t_stopf('docn_initgsmaps') !---------------------------------------------------------------------------- - ! Initialize MCT domain + ! Initialize data model MCT domain !---------------------------------------------------------------------------- call t_startf('docn_initmctdom') if (my_task == master_task) write(logunit,F00) 'copy domains' call shr_sys_flush(logunit) - if (ocn_present) call shr_dmodel_rearrGGrid(SDOCN%grid, ggrid, gsmap, rearr, mpicom) + if (ocn_present) then + call shr_dmodel_rearrGGrid(SDOCN%grid, ggrid, gsmap, rearr, mpicom) + end if + + ! Special logic for either prescribed or som aquaplanet - overwrite and + ! set mask/frac to 1 + if (ocn_mode == 'SST_AQUAP' .or. ocn_mode == 'SOM_AQUAP') then + kmask = mct_aVect_indexRA(ggrid%data,'mask') + ggrid%data%rattr(kmask,:) = 1 + kfrac = mct_aVect_indexRA(ggrid%data,'frac') + ggrid%data%rattr(kfrac,:) = 1.0_r8 + write(logunit,F00) ' Resetting the data ocean mask and frac to 1 for aquaplanet' + end if call t_stopf('docn_initmctdom') @@ -426,10 +459,18 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) allocate(somtp(lsize)) allocate(tfreeze(lsize)) allocate(imask(lsize)) + allocate(xc(lsize)) + allocate(yc(lsize)) kmask = mct_aVect_indexRA(ggrid%data,'mask') imask(:) = nint(ggrid%data%rAttr(kmask,:)) + index_lon = mct_aVect_indexRA(ggrid%data,'lon') + xc(:) = ggrid%data%rAttr(index_lon,:) + + index_lat = mct_aVect_indexRA(ggrid%data,'lat') + yc(:) = ggrid%data%rAttr(index_lat,:) + call t_stopf('docn_initmctavs') !---------------------------------------------------------------------------- @@ -466,7 +507,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) endif endif call shr_mpi_bcast(exists,mpicom,'exists') - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) call shr_pcdf_readwrite('read',iosystem,SDOCN%io_type,trim(rest_file),mpicom,gsmap,rf1=somtp,rf1n='somtp') endif @@ -539,7 +580,6 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) integer(IN) :: nl ! ocn frac index integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level - logical :: glcrun_alarm ! is glc going to run now logical :: newdata ! has newdata been read logical :: mssrmlf ! remove old data integer(IN) :: idt ! integer timestep @@ -583,23 +623,6 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) call t_stopf('docn_run1') - !-------------------- - ! UNPACK - !-------------------- - - call t_startf('docn_unpack') - -! lsize = mct_avect_lsize(x2o) -! nflds_x2o = mct_avect_nRattr(x2o) - -! do nf=1,nflds_x2o -! do n=1,lsize -! ?? = x2o%rAttr(nf,n) -! enddo -! enddo - - call t_stopf('docn_unpack') - !-------------------- ! ADVANCE OCN !-------------------- @@ -658,6 +681,16 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) o2x%rAttr(kswp ,n) = swp enddo + case('SST_AQUAP') + lsize = mct_avect_lsize(o2x) + do n = 1,lsize + o2x%rAttr(:,n) = 0.0_r8 + end do + call prescribed_sst(xc, yc, lsize, aquap_option, o2x%rAttr(kt,:)) + do n = 1,lsize + o2x%rAttr(kt,n) = o2x%rAttr(kt,n) + TkFrz + enddo + case('IAF') lsize = mct_avect_lsize(o2x) do n = 1,lsize @@ -709,6 +742,41 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) enddo endif ! firstcall + case('SOM_AQUAP') + lsize = mct_avect_lsize(o2x) + do n = 1,SDOCN%nstreams + call shr_dmodel_translateAV(SDOCN%avs(n),avstrm,avifld,avofld,rearr) + enddo + if (firstcall) then + do n = 1,lsize + if (.not. read_restart) then + somtp(n) = o2x%rAttr(kt,n) + TkFrz + endif + o2x%rAttr(kt,n) = somtp(n) + o2x%rAttr(kq,n) = 0.0_R8 + enddo + else ! firstcall + tfreeze = shr_frz_freezetemp(o2x%rAttr(ks,:)) + TkFrz + do n = 1,lsize + !--- pull out h from av for resuse below --- + hn = avstrm%rAttr(kh,n) + !--- compute new temp --- + o2x%rAttr(kt,n) = somtp(n) + & + (x2o%rAttr(kswnet,n) + & ! shortwave + x2o%rAttr(klwup ,n) + & ! longwave + x2o%rAttr(klwdn ,n) + & ! longwave + x2o%rAttr(ksen ,n) + & ! sensible + x2o%rAttr(klat ,n) + & ! latent + x2o%rAttr(kmelth,n) - & ! ice melt + avstrm%rAttr(kqbot ,n) - & ! flux at bottom + (x2o%rAttr(ksnow,n)+x2o%rAttr(krofi,n))*latice) * & ! latent by prec and roff + dt/(cpsw*rhosw*hn) + !--- compute ice formed or melt potential --- + o2x%rAttr(kq,n) = (tfreeze(n) - o2x%rAttr(kt,n))*(cpsw*rhosw*hn)/dt ! ice formed q>0 + somtp(n) = o2x%rAttr(kt,n) ! save temp + enddo + endif ! firstcall + end select call t_stopf('docn_mode') @@ -730,7 +798,7 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) close(nu) call shr_file_freeUnit(nu) endif - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then if (my_task == master_task) write(logunit,F04) ' writing ',trim(rest_file),currentYMD,currentTOD call shr_pcdf_readwrite('write',iosystem,SDOCN%io_type,trim(rest_file),mpicom,gsmap,clobber=.true., & rf1=somtp,rf1n='somtp') @@ -801,8 +869,213 @@ subroutine docn_comp_final() call t_stopf('DOCN_FINAL') end subroutine docn_comp_final + !=============================================================================== !=============================================================================== +subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) + + real(R8) , intent(in) :: xc(:) !degrees + real(R8) , intent(in) :: yc(:) !degrees + integer(IN) , intent(in) :: lsize + integer(IN) , intent(in) :: sst_option + real(R8) , intent(inout) :: sst(:) + + ! local + integer :: i + real(r8) :: tmp, tmp1, pi + real(r8) :: rlon(lsize), rlat(lsize) + + real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 + + ! Parameters for zonally symmetric experiments + real(r8), parameter :: t0_max = 27._r8 + real(r8), parameter :: t0_min = 0._r8 + real(r8), parameter :: maxlat = 60._r8*pio180 + real(r8), parameter :: shift = 5._r8*pio180 + real(r8), parameter :: shift9 = 10._r8*pio180 + real(r8), parameter :: shift10 = 15._r8*pio180 + + ! Parameters for zonally asymmetric experiments + real(r8), parameter :: t0_max6 = 1._r8 + real(r8), parameter :: t0_max7 = 3._r8 + real(r8), parameter :: latcen = 0._r8*pio180 + real(r8), parameter :: loncen = 0._r8*pio180 + real(r8), parameter :: latrad6 = 15._r8*pio180 + real(r8), parameter :: latrad8 = 30._r8*pio180 + real(r8), parameter :: lonrad = 30._r8*pio180 + !------------------------------------------------------------------------------- + + pi = SHR_CONST_PI + + ! convert xc and yc from degrees to radians + + rlon(:) = xc(:) * pio180 + rlat(:) = yc(:) * pio180 + + ! Control + + if (sst_option < 1 .or. sst_option > 10) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + end if + + if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Flat + + if (sst_option == 2) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = 1._r8 - tmp*tmp*tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Qobs + + if (sst_option == 3) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = (2._r8 - tmp*tmp*tmp*tmp - tmp*tmp)*0.5_r8 + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Peaked + + if (sst_option == 4) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = (maxlat - abs(rlat(i)))/maxlat + tmp1 = 1._r8 - tmp + sst(i) = t0_max*tmp + t0_min*tmp1 + end if + end do + end if + + ! Control-5N + + if (sst_option == 5) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if (rlat(i) > shift) then + tmp = sin((rlat(i)-shift)*pi*0.5_r8/(maxlat-shift)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift)*pi*0.5_r8/(maxlat+shift)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! 1KEQ + + if (sst_option == 6) then + do i = 1,lsize + if (abs(rlat(i)-latcen) <= latrad6) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad6) + tmp1 = tmp1*tmp1 + tmp = abs(rlon(i)-loncen) + tmp = min(tmp , 2._r8*pi-tmp) + if(tmp <= lonrad) then + tmp = cos(tmp*pi*0.5_r8/lonrad) + tmp = tmp*tmp + sst(i) = sst(i) + t0_max6*tmp*tmp1 + end if + end if + end do + end if + + ! 3KEQ + + if (sst_option == 7) then + do i = 1, lsize + if (abs(rlat(i)-latcen) <= latrad6) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad6) + tmp1 = tmp1*tmp1 + tmp = abs(rlon(i)-loncen) + tmp = min(tmp , 2._r8*pi-tmp) + if (tmp <= lonrad) then + tmp = cos(tmp*pi*0.5_r8/lonrad) + tmp = tmp*tmp + sst(i) = sst(i) + t0_max7*tmp*tmp1 + end if + end if + end do + end if + + ! 3KW1 + + if (sst_option == 8) then + do i = 1, lsize + if (abs(rlat(i)-latcen) <= latrad8) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad8) + tmp1 = tmp1*tmp1 + tmp = cos(rlon(i)-loncen) + sst(i) = sst(i) + t0_max7*tmp*tmp1 + end if + end do + end if + + ! Control-10N + + if (sst_option == 9) then + do i = 1, lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if (rlat(i) > shift9) then + tmp = sin((rlat(i)-shift9)*pi*0.5_r8/(maxlat-shift9)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift9)*pi*0.5_r8/(maxlat+shift9)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Control-15N + + if (sst_option == 10) then + do i = 1, lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if(rlat(i) > shift10) then + tmp = sin((rlat(i)-shift10)*pi*0.5_r8/(maxlat-shift10)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift10)*pi*0.5_r8/(maxlat+shift10)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + +end subroutine prescribed_sst end module docn_comp_mod diff --git a/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml index a5a953ba8ed6..03a819b67f16 100644 --- a/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -212,7 +212,7 @@ char streams shr_strdata_nml - NULL,CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1 + NULL,COPYALL The runoff data is associated with the river model. datamode = "NULL" @@ -223,21 +223,10 @@ dataMode = "COPYALL" Copies all fields directly from the input data streams Any required fields not found on an input stream will be set to zero. - datamode = "CPLHIST" - Reads in data from coupler history files generated by a previous run - datamode = DIATREN_ANN_RX1 - Reads in annual forcing river data used for CORE2 forcing runs - datamode = DIATREN_ANN_RX1,DIATREN_IAF_RX1 - Reads in intra-annual forcing river data used for CORE2 forcing runs - Set by the the DROF_MODE xml variables in env_run.xml - specifies values for rof mode: CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1,NULL - default value: DIATREN_ANN_RX1 - NULL - CPLHIST - DIATREN_ANN_RX1 - DIATREN_IAF_RX1 + COPYALL + NULL diff --git a/src/components/data_comps/drof/drof_comp_mod.F90 b/src/components/data_comps/drof/drof_comp_mod.F90 index 060b5fad678c..a2c9974bd3ac 100644 --- a/src/components/data_comps/drof/drof_comp_mod.F90 +++ b/src/components/data_comps/drof/drof_comp_mod.F90 @@ -255,10 +255,7 @@ subroutine drof_comp_init( EClock, cdata, x2r, r2x, NLFilename ) ! check that we know how to handle the mode - if (trim(rof_mode) == 'NULL' .or. & - trim(rof_mode) == 'CPLHIST' .or. & - trim(rof_mode) == 'DIATREN_ANN_RX1' .or. & - trim(rof_mode) == 'DIATREN_IAF_RX1') then + if (trim(rof_mode) == 'NULL' .or. trim(rof_mode) == 'COPYALL') then if (my_task == master_task) write(logunit,F00) 'rof mode = ',trim(rof_mode) else write(logunit,F00) ' ERROR illegal rof mode = ',trim(rof_mode) diff --git a/src/components/data_comps/dwav/cime_config/buildnml b/src/components/data_comps/dwav/cime_config/buildnml index 7a0d67cee2c3..3aeffe5afbb3 100755 --- a/src/components/data_comps/dwav/cime_config/buildnml +++ b/src/components/data_comps/dwav/cime_config/buildnml @@ -1,7 +1,6 @@ #!/usr/bin/env python -"""Namelist creator for CIME's data ocn model. -`build_namelist` function. +"""Namelist creator for CIME's data wave model. """ # Typically ignore this. # pylint: disable=invalid-name @@ -9,10 +8,7 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os -import shutil -import sys -import glob +import os, shutil, sys, glob _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) @@ -63,8 +59,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): #---------------------------------------------------- # Clear out old data. #---------------------------------------------------- - data_list_path = os.path.join(case.get_case_root(), "Buildconf", - "dwav.input_data_list") + data_list_path = os.path.join(case.get_case_root(), "Buildconf", "dwav.input_data_list") if os.path.exists(data_list_path): os.remove(data_list_path) @@ -97,8 +92,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): inst_stream = stream + inst_string logger.debug("DWAV stream is %s", inst_stream) - stream_path = os.path.join(confdir, - "dwav.streams.txt." + inst_stream) + stream_path = os.path.join(confdir, "dwav.streams.txt." + inst_stream) user_stream_path = os.path.join(case.get_case_root(), "user_dwav.streams.txt." + inst_stream) @@ -113,11 +107,10 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): #---------------------------------------------------- # Create dwav_nml namelists group #---------------------------------------------------- + # set per-stream variables nmlgen.create_shr_strdata_nml() - nmlgen.add_default("decomp", "1d") - nmlgen.add_default("force_prognostic_true", value=".false.") - nmlgen.add_default("restfilm", value="undefined") - nmlgen.add_default("restfils", value="undefined") + + # set variables that are not per-stream if wav_domain_file != "UNSET": full_domain_path = os.path.join(wav_domain_path, wav_domain_file) nmlgen.add_default("domainfile", value=full_domain_path) @@ -126,7 +119,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # Finally, write out all the namelists. #---------------------------------------------------- namelist_file = os.path.join(confdir, "dwav_in") - nmlgen.write_output_file(namelist_file, data_list_path) + nmlgen.write_output_file(namelist_file, data_list_path, groups=['dwav_nml','shr_strdata_nml']) ############################################################################### def buildnml(case, caseroot, compname): diff --git a/src/components/data_comps/dwav/cime_config/config_component.xml b/src/components/data_comps/dwav/cime_config/config_component.xml index a7fb6c1c0558..3b73827e7fdf 100644 --- a/src/components/data_comps/dwav/cime_config/config_component.xml +++ b/src/components/data_comps/dwav/cime_config/config_component.xml @@ -13,11 +13,11 @@ char - null,copyall - copyall + NULL,CLIMO + NULL - null - copyall + NULL + CLIMO run_component_dwav env_run.xml @@ -28,7 +28,7 @@ dwav null mode: - dwav copy mode: + dwav climatological mode: diff --git a/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml b/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml index c2f12ca9b4e6..81398a316f7a 100644 --- a/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +++ b/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml @@ -47,10 +47,10 @@ char(100) streams streams_file - List of streams used for the given datm_mode. + List of streams used for the given dwav_mode. - NULL - wav.copyall + null + climo @@ -60,7 +60,7 @@ streams_file Stream domain file directory. - $DIN_LOC_ROOT/wav/dwav + $DIN_LOC_ROOT/wav/dwav @@ -70,7 +70,7 @@ streams_file Stream domain file path(s). - waveclim.mon.ww3a.150612.nc + waveclim.mon.ww3a.150612.nc @@ -80,7 +80,7 @@ streams_file Stream domain variable name(s). - + time time xc lon yc lat @@ -96,7 +96,7 @@ streams_file Stream data file directory. - $DIN_LOC_ROOT/wav/dwav + $DIN_LOC_ROOT/wav/dwav @@ -106,7 +106,7 @@ streams_file Stream data file path(s). - waveclim.mon.ww3a.150612.nc + waveclim.mon.ww3a.150612.nc @@ -116,7 +116,7 @@ streams_file Stream data variable name(s). - + lamult lamult ustokes ustokes vstokes vstokes @@ -140,7 +140,7 @@ streams_file Simulation year to align stream to. - 1 + 1 @@ -150,7 +150,7 @@ streams_file First year of stream. - 1 + 1 @@ -160,7 +160,7 @@ streams_file Last year of stream. - 1 + 1 @@ -188,15 +188,14 @@ fields not found on an input stream will be set to zero. Set by the following xml variables in env_run.xml DWAV_MODE specifies values for wav mode: copyall,null - default value: copyall - null - copyall + copyall + null - + char streams abs @@ -207,7 +206,7 @@ for all input data for this strdata input. - + null @@ -253,7 +252,7 @@ and fillmask are ignored. - + NOT_SET @@ -268,7 +267,7 @@ save and reuse a set of weights later. - + NOT_SET @@ -317,7 +316,7 @@ is set, mapalgo and mapmask are ignored. - + NOT_SET @@ -332,7 +331,7 @@ allows a user to save and reuse a set of weights later. - + NOT_SET @@ -401,7 +400,7 @@ - + char streams shr_strdata_nml @@ -483,8 +482,8 @@ logical - drof - drof_nml + dwav + dwav_nml If TRUE, prognostic is forced to true. .false. diff --git a/src/components/data_comps/dwav/dwav_comp_mod.F90 b/src/components/data_comps/dwav/dwav_comp_mod.F90 index 4c45d364d4a2..f33019887975 100644 --- a/src/components/data_comps/dwav/dwav_comp_mod.F90 +++ b/src/components/data_comps/dwav/dwav_comp_mod.F90 @@ -1,3 +1,6 @@ +#ifdef AIX +@PROCESS ALIAS_SIZE(805306368) +#endif module dwav_comp_mod ! !USES: @@ -51,9 +54,8 @@ module dwav_comp_mod integer(IN),parameter :: master_task=0 ! task number of master task integer(IN) :: logunit ! logging unit number integer :: inst_index ! number of current instance (ie. 1) - character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") - character(len=16) :: inst_suffix ! char string associated with instance - ! (ie. "_0001" or "") + character(len=16) :: inst_name ! fullname of current instance (ie. "wav_0001") + character(len=16) :: inst_suffix ! char string associated with instance (ie. "_0001" or "") character(CL) :: wav_mode ! mode integer(IN) :: dbug = 0 ! debug level (higher is more) @@ -66,10 +68,8 @@ module dwav_comp_mod type(mct_rearr) :: rearr integer(IN),parameter :: ktrans = 3 - character(12),parameter :: avifld(1:ktrans) = & - (/"lamult ","ustokes ","vstokes "/) - character(12),parameter :: avofld(1:ktrans) = & - (/"Sw_lamult ","Sw_ustokes ","Sw_vstokes "/) + character(12),parameter :: avifld(1:ktrans) = (/"lamult ","ustokes ","vstokes "/) + character(12),parameter :: avofld(1:ktrans) = (/"Sw_lamult ","Sw_ustokes ","Sw_vstokes "/) !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS @@ -128,21 +128,25 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) integer(IN) :: yearLast ! last year to use in data stream integer(IN) :: yearAlign ! data year that aligns with yearFirst - character(CL) :: wav_in ! dshr wav namelist - character(CL) :: decomp ! decomp strategy - character(CL) :: rest_file ! restart filename - character(CL) :: rest_file_strm ! restart filename for stream - character(CL) :: restfilm ! restart filename for namelist - character(CL) :: restfils ! restart filename for stream for namelist + character(CL) :: wav_in ! dshr wav namelist + character(CL) :: decomp ! decomp strategy + character(CL) :: rest_file ! restart filename + character(CL) :: rest_file_strm ! restart filename for stream + character(CL) :: restfilm ! restart filename for namelist + character(CL) :: restfils ! restart filename for stream for namelist + logical :: force_prognostic_true ! if true set prognostic true + logical :: exists ! file existance integer(IN) :: nu ! unit number !----- define namelist ----- namelist / dwav_nml / & - decomp, restfilm, restfils + decomp, restfilm, restfils, & + force_prognostic_true !--- formats --- character(*), parameter :: F00 = "('(dwav_comp_init) ',8a)" + character(*), parameter :: F0L = "('(docn_comp_init) ',a, l2)" character(*), parameter :: F01 = "('(dwav_comp_init) ',a,5i8)" character(*), parameter :: F02 = "('(dwav_comp_init) ',a,4es13.6)" character(*), parameter :: F03 = "('(dwav_comp_init) ',a,i8,a)" @@ -151,8 +155,7 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) character(*), parameter :: F90 = "('(dwav_comp_init) ',73('='))" character(*), parameter :: F91 = "('(dwav_comp_init) ',73('-'))" character(*), parameter :: subName = "(dwav_comp_init) " - -!------------------------------------------------------------------------------- + !------------------------------------------------------------------------------- call t_startf('DWAV_INIT') @@ -202,7 +205,6 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) !---------------------------------------------------------------------------- call t_startf('dwav_readnml') - !write(logunit,F00)' dwav_readnml...' filename = "dwav_in"//trim(inst_suffix) decomp = "1d" @@ -218,35 +220,37 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) write(logunit,F01) 'ERROR: reading input namelist, '//trim(filename)//' iostat=',ierr call shr_sys_abort(subName//': namelist read error '//trim(filename)) end if - write(logunit,F00)' wav_in = ',trim(wav_in) write(logunit,F00)' decomp = ',trim(decomp) write(logunit,F00)' restfilm = ',trim(restfilm) write(logunit,F00)' restfils = ',trim(restfils) + write(logunit,F0L)' force_prognostic_true = ',force_prognostic_true endif call shr_mpi_bcast(decomp,mpicom,'decomp') call shr_mpi_bcast(restfilm,mpicom,'restfilm') call shr_mpi_bcast(restfils,mpicom,'restfils') + call shr_mpi_bcast(force_prognostic_true,mpicom,'force_prognostic_true') rest_file = trim(restfilm) rest_file_strm = trim(restfils) + if (force_prognostic_true) then + wav_present = .true. + wav_prognostic = .true. + endif !---------------------------------------------------------------------------- ! Read dshr namelist !---------------------------------------------------------------------------- - !write(logunit,F00)' read dshr nml...' call shr_strdata_readnml(SDWAV,trim(filename),mpicom=mpicom) !---------------------------------------------------------------------------- ! Validate mode !---------------------------------------------------------------------------- - !write(logunit,F00)' validate mode...' - wav_mode = trim(SDWAV%dataMode) ! check that we know how to handle the mode - if (trim(wav_mode) == 'null' .or. & + if (trim(wav_mode) == 'null' .or. & trim(wav_mode) == 'copyall') then if (my_task == master_task) & write(logunit,F00) ' wav mode = ',trim(wav_mode) @@ -262,7 +266,6 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) !---------------------------------------------------------------------------- call t_startf('dwav_strdata_init') - !write(logunit,F00)' dwav_strdata_init...' if (trim(wav_mode) /= 'null') then wav_present = .true. @@ -271,8 +274,7 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) call shr_strdata_pioinit(SDWAV, iosystem, shr_pio_getiotype(trim(inst_name))) - call shr_strdata_init(SDWAV,mpicom,compid,name='wav', & - calendar=calendar) + call shr_strdata_init(SDWAV,mpicom,compid,name='wav', calendar=calendar) endif if (my_task == master_task) then @@ -440,25 +442,20 @@ subroutine dwav_comp_run( EClock, cdata, x2w, w2x) type(mct_gsMap) , pointer :: gsmap type(mct_gGrid) , pointer :: ggrid - integer(IN) :: CurrentYMD ! model date - integer(IN) :: CurrentTOD ! model sec into model date - integer(IN) :: yy,mm,dd ! year month day - integer(IN) :: n ! indices - integer(IN) :: nf ! fields loop index - integer(IN) :: nl ! ocn frac index - integer(IN) :: lsize ! size of attr vect + integer(IN) :: CurrentYMD ! model date + integer(IN) :: CurrentTOD ! model sec into model date + integer(IN) :: yy,mm,dd ! year month day + integer(IN) :: n ! indices + integer(IN) :: nf ! fields loop index + integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level -! logical :: glcrun_alarm ! is glc going to run now - logical :: newdata ! has newdata been read - logical :: mssrmlf ! remove old data - integer(IN) :: idt ! integer timestep - real(R8) :: dt ! timestep -! real(R8) :: hn ! h field - logical :: write_restart ! restart now - character(CL) :: case_name ! case name - character(CL) :: rest_file ! restart_file - character(CL) :: rest_file_strm ! restart_file for stream - integer(IN) :: nu ! unit number + integer(IN) :: idt ! integer timestep + real(R8) :: dt ! timestep + logical :: write_restart ! restart now + character(CL) :: case_name ! case name + character(CL) :: rest_file ! restart_file + character(CL) :: rest_file_strm ! restart_file for stream + integer(IN) :: nu ! unit number integer(IN) :: nflds_x2w type(seq_infodata_type), pointer :: infodata @@ -494,14 +491,14 @@ subroutine dwav_comp_run( EClock, cdata, x2w, w2x) call t_startf('dwav_unpack') -! lsize = mct_avect_lsize(x2o) -! nflds_x2o = mct_avect_nRattr(x2o) + ! lsize = mct_avect_lsize(x2o) + ! nflds_x2o = mct_avect_nRattr(x2o) -! do nf=1,nflds_x2o -! do n=1,lsize -! ?? = x2o%rAttr(nf,n) -! enddo -! enddo + ! do nf=1,nflds_x2o + ! do n=1,lsize + ! ?? = x2o%rAttr(nf,n) + ! enddo + ! enddo call t_stopf('dwav_unpack') diff --git a/src/drivers/mct/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe index fbf07f1feb9a..627330611a92 100755 --- a/src/drivers/mct/cime_config/buildexe +++ b/src/drivers/mct/cime_config/buildexe @@ -31,8 +31,12 @@ def _main_func(): gmake = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") model = case.get_value("MODEL") + num_esp = case.get_value("NUM_COMP_INST_ESP") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) + expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") + + with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.drv") + "\n") out.write(os.path.join(cimeroot, "src", "drivers", "mct", "main") + "\n") diff --git a/src/drivers/mct/cime_config/buildnml b/src/drivers/mct/cime_config/buildnml index 71f687381414..c2076dcad0b2 100755 --- a/src/drivers/mct/cime_config/buildnml +++ b/src/drivers/mct/cime_config/buildnml @@ -16,7 +16,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.nmlgen import NamelistGenerator from CIME.utils import expect -from CIME.utils import run_cmd_no_fail, get_model +from CIME.utils import run_cmd_no_fail, get_model, get_time_in_seconds from CIME.buildnml import create_namelist_infile, parse_input from CIME.XML.files import Files @@ -93,6 +93,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): expect(False, "basedt invalid overflow for NCPL_BASE_PERIOD %s " %ncpl_base_period) comps = case.get_values("COMP_CLASSES") + mindt = basedt for comp in comps: ncpl = case.get_value(comp.upper() + '_NCPL') if ncpl is not None: @@ -100,7 +101,8 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): totaldt = cpl_dt * int(ncpl) if totaldt != basedt: expect(False, " %s ncpl doesn't divide base dt evenly" %comp) - nmlgen.set_value(comp.lower() + '_cpl_dt', value=cpl_dt) + nmlgen.add_default(comp.lower() + '_cpl_dt', value=cpl_dt) + mindt = min(mindt, cpl_dt) # elif comp.lower() is not 'cpl': #-------------------------------- @@ -142,6 +144,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # Set up the pause_component_list if pause is active pauseo = case.get_value('PAUSE_OPTION') if pauseo != 'never' and pauseo != 'none': + pausen = case.get_value('PAUSE_N') pcl = nmlgen.get_default('pause_component_list') nmlgen.add_default('pause_component_list', pcl) # Check to make sure pause_component_list is valid @@ -154,6 +157,13 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): "Invalid PAUSE_COMPONENT_LIST, %s is not a valid component type"%comp) # End for # End if + # Set esp interval + if 'nstep' in pauseo: + esp_time = mindt + else: + esp_time = get_time_in_seconds(pausen, pauseo) + + nmlgen.set_value('esp_cpl_dt', value=esp_time) # End if pause is active #-------------------------------- diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index aa092aa3e600..f46a999938c7 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -303,7 +303,7 @@ run_begin_stop_restart env_run.xml - Sets the run length along with STOP_N and STOP_DATE (must be nyear(s) for _GLC compsets for restarts to work properly). + Sets the run length along with STOP_N and STOP_DATE @@ -335,8 +335,7 @@ run_begin_stop_restart env_run.xml - sets frequency of model restart writes (same options as STOP_OPTION) (must be nyear(s) for _GLC compsets) - (must be nyear(s) for _GLC compsets) + sets frequency of model restart writes (same options as STOP_OPTION) @@ -368,7 +367,7 @@ run_begin_stop_restart env_run.xml - Sets the pause frequency along with PAUSE_N (must be nyear(s) for _GLC compsets for restarts to resume properly). + Sets the pause frequency along with PAUSE_N @@ -418,6 +417,23 @@ + + logical + TRUE,FALSE + FALSE + run_begin_stop_restart + env_run.xml + + ESP component runs after driver 'pause cycle' + If any component 'pauses' (see PAUSE_OPTION, PAUSE_N and + PAUSE_COMPONENT_LIST XML variables), the ESP component (if + present) will be run to process the component 'pause' (restart) + files and set any required 'resume' signals. + If true, esp_cpl_dt and esp_cpl_offset settings are ignored. + default: false + + + logical TRUE,FALSE @@ -1890,7 +1906,7 @@ UNSET run_mpi env_run.xml - mpi run command + override the mpi run command, do not include model executable @@ -2566,6 +2582,15 @@ The machine wallclock setting. Default determined in config_machines.xml can be overwritten by testing + + char + + + job_submission + env_batch.xml + Override the batch submit command this job. Do not include executable or dependencies + + char diff --git a/src/drivers/mct/cime_config/config_component_acme.xml b/src/drivers/mct/cime_config/config_component_acme.xml index 8350eb7084b1..cf7693de75c6 100644 --- a/src/drivers/mct/cime_config/config_component_acme.xml +++ b/src/drivers/mct/cime_config/config_component_acme.xml @@ -374,6 +374,24 @@ Number of glc coupling intervals per NCPL_BASE_PERIOD. + + char + glc_coupling_period,yearly + glc_coupling_period + run_coupling + env_run.xml + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting GLC_AVG_PERIOD to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + + + integer 8 diff --git a/src/drivers/mct/cime_config/config_component_cesm.xml b/src/drivers/mct/cime_config/config_component_cesm.xml index 7195dee649b4..a14438fdc1ce 100644 --- a/src/drivers/mct/cime_config/config_component_cesm.xml +++ b/src/drivers/mct/cime_config/config_component_cesm.xml @@ -209,6 +209,27 @@ Number of glc coupling intervals per NCPL_BASE_PERIOD. + + char + glc_coupling_period,yearly + yearly + run_coupling + env_run.xml + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting GLC_AVG_PERIOD to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + + IMPORTANT: In order to restart mid-year when running with CISM, you MUST specify GLC_AVG_PERIOD = 'yearly'. + If using GLC_AVG_PERIOD = 'glc_coupling_period' with CISM, you can only restart on year boundaries. + + + integer 8 diff --git a/src/drivers/mct/cime_config/config_compsets.xml b/src/drivers/mct/cime_config/config_compsets.xml index c11354e14503..fb129089b514 100644 --- a/src/drivers/mct/cime_config/config_compsets.xml +++ b/src/drivers/mct/cime_config/config_compsets.xml @@ -9,19 +9,22 @@ The compset longname below has the specified order atm, lnd, ice, ocn, river, glc wave esp cesm-options - The notation for the compset longname is + The notation for the compset longname below is TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP][_BGC%phys] - Where for the CAM specific compsets below the following is supported + + The following compsets are those that can be tested in CIME stand-alone configurations + without any prognostic components. + For the compsets below the following are the only allowable values of the components. + TIME = Time period (e.g. 2000, HIST, RCP8...) - ATM = [CAM4, CAM5] - LND = [CLM40, CLM45, CLM50, SLND] - ICE = [CICE, DICE, SICE] - OCN = [DOCN, ,AQUAP, SOCN] - ROF = [RTM, SROF] - GLC = [CISM1, CISM2, SGLC] - WAV = [SWAV WW3] - ESP = [SESP DESP] - BGC = optional BGC scenario + ATM = [DATM, SATM, XATM] + LND = [DLND, SLND, XLND] + ICE = [DICE, SICE, XICE] + OCN = [DOCN, SOCN, XOCN] + ROF = [DROF, SROF, XROF] + GLC = [ SGLC ] + WAV = [DWAV, SWAV ] + ESP = [DESP, SESP ] The OPTIONAL %phys attributes specify submodes of the given system For example DOCN%DOM is the data ocean model for DOCN @@ -42,6 +45,16 @@ 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV + + ADLND + 2000_SATM_DLND%SCPL_SICE_SOCN_SROF_SGLC_SWAV + + + + ADWAV + 2000_SATM_SLND_SICE_SOCN_SROF_SGLC_DWAV%CLIMO + + ADESP 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_DESP @@ -57,11 +70,6 @@ 2000_DATM%IAF_SLND_DICE%IAF_DOCN%IAF_DROF%IAF_SGLC_SWAV - - AWAV - 2000_DATM%WW3_SLND_DICE%COPY_DOCN%COPY_SROF_SGLC_WW3 - - S 2000_SATM_SLND_SICE_SOCN_SROF_SGLC_SWAV_SESP diff --git a/src/drivers/mct/cime_config/namelist_definition_drv.xml b/src/drivers/mct/cime_config/namelist_definition_drv.xml index bffe785aef28..82a7d634e783 100644 --- a/src/drivers/mct/cime_config/namelist_definition_drv.xml +++ b/src/drivers/mct/cime_config/namelist_definition_drv.xml @@ -1,6 +1,6 @@ - + @@ -352,7 +352,7 @@ - + logical expdef seq_infodata_inparm @@ -1387,7 +1387,7 @@ - + integer time seq_timemgr_inparm @@ -1398,12 +1398,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, and has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1414,12 +1411,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1428,12 +1422,9 @@ currently set by default to 10800 seconds. default: 10800 - - 10800 - - + integer time seq_timemgr_inparm @@ -1444,12 +1435,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1460,12 +1448,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1476,12 +1461,30 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + + + + char + time + seq_timemgr_inparm + glc_coupling_period,yearly - -999 + $GLC_AVG_PERIOD + + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting glc_avg_period to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + - + integer time seq_timemgr_inparm @@ -1492,6 +1495,20 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + + + + integer + time + seq_timemgr_inparm + + esp run interval in seconds + esp_cpl_dt is the number of times the esp is run per NCPL_BASE_PERIOD + NCPL_BASE_PERIOD is set in env_run.xml and is the base period + associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + default value set by buildnml to be the pause interval if pause is active + otherwise, it is set to the shortest component coupling time + -999 @@ -1569,6 +1586,36 @@ + + integer + time + seq_timemgr_inparm + + esp coupling interval offset in seconds default: 0 + + + 0 + + + + + logical + time + seq_timemgr_inparm + + true => ESP component runs after driver 'pause cycle' + If any component 'pauses' (see PAUSE_OPTION, PAUSE_N and + PAUSE_COMPONENT_LIST XML variables), the ESP component (if + present) will be run to process the component 'pause' (restart) + files and set any required 'resume' signals. + If true, esp_cpl_dt and esp_cpl_offset settings are ignored. + default: true + + + .true. + + + char time diff --git a/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml index 1a9cc9376b8e..08847103afde 100644 --- a/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml +++ b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml @@ -86,6 +86,19 @@ + + + + + + char(2) + nitrogen deposition + ndep_inparm + + List of nitrogen deposition fluxes to be sent from CAM to surfae models. + + + diff --git a/src/drivers/mct/main/cesm_comp_mod.F90 b/src/drivers/mct/main/cesm_comp_mod.F90 index afc6b863ff4f..f9a8637fa1ad 100644 --- a/src/drivers/mct/main/cesm_comp_mod.F90 +++ b/src/drivers/mct/main/cesm_comp_mod.F90 @@ -91,6 +91,7 @@ module cesm_comp_mod use seq_timemgr_mod, only: seq_timemgr_alarm_ocnrun use seq_timemgr_mod, only: seq_timemgr_alarm_icerun use seq_timemgr_mod, only: seq_timemgr_alarm_glcrun + use seq_timemgr_mod, only: seq_timemgr_alarm_glcrun_avg use seq_timemgr_mod, only: seq_timemgr_alarm_ocnnext use seq_timemgr_mod, only: seq_timemgr_alarm_tprof use seq_timemgr_mod, only: seq_timemgr_alarm_histavg @@ -99,13 +100,15 @@ module cesm_comp_mod use seq_timemgr_mod, only: seq_timemgr_alarm_esprun use seq_timemgr_mod, only: seq_timemgr_alarm_barrier use seq_timemgr_mod, only: seq_timemgr_alarm_pause - use seq_timemgr_mod, only: pause_component_list=>seq_timemgr_pause_component_list + use seq_timemgr_mod, only: seq_timemgr_pause_active + use seq_timemgr_mod, only: seq_timemgr_pause_component_active + use seq_timemgr_mod, only: seq_timemgr_pause_component_index ! "infodata" gathers various control flags into one datatype use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData use seq_infodata_mod, only: seq_infodata_init, seq_infodata_exchange use seq_infodata_mod, only: seq_infodata_type, seq_infodata_orb_variable_year - use seq_infodata_mod, only: seq_infodata_print + use seq_infodata_mod, only: seq_infodata_print, seq_infodata_init2 ! domain related routines use seq_domain_mct, only : seq_domain_check @@ -252,12 +255,12 @@ module cesm_comp_mod logical :: ocnrun_alarm ! ocn run alarm logical :: ocnnext_alarm ! ocn run alarm on next timestep logical :: glcrun_alarm ! glc run alarm + logical :: glcrun_avg_alarm ! glc run averaging alarm logical :: rofrun_alarm ! rof run alarm logical :: wavrun_alarm ! wav run alarm logical :: esprun_alarm ! esp run alarm logical :: tprof_alarm ! timing profile alarm logical :: barrier_alarm ! barrier alarm - logical :: pause_alarm ! pause alarm logical :: t1hr_alarm ! alarm every hour logical :: t2hr_alarm ! alarm every two hours logical :: t3hr_alarm ! alarm every three hours @@ -265,6 +268,8 @@ module cesm_comp_mod logical :: t12hr_alarm ! alarm every twelve hours logical :: t24hr_alarm ! alarm every twentyfour hours logical :: t1yr_alarm ! alarm every year, at start of year + logical :: pause_alarm ! pause alarm + integer :: drv_index ! seq_timemgr index for driver real(r8) :: days_per_year = 365.0 ! days per year @@ -1002,6 +1007,11 @@ subroutine cesm_pre_init2() call seq_timemgr_clockPrint(seq_SyncClock) endif + !---------------------------------------------------------- + !| Initialize infodata items which need the clocks + !---------------------------------------------------------- + call seq_infodata_init2(infodata, GLOID) + call seq_infodata_getData(infodata, & orb_iyear=orb_iyear, & orb_iyear_align=orb_iyear_align, & @@ -1966,6 +1976,11 @@ subroutine cesm_init() call t_stopf('comp_init_cc_atm2') endif ! atm present + !---------------------------------------------------------- + !| Get time manager's index for driver + !---------------------------------------------------------- + drv_index = seq_timemgr_pause_component_index('drv') + !---------------------------------------------------------- !| Read driver restart file, overwrite anything previously sent or computed !---------------------------------------------------------- @@ -2052,8 +2067,8 @@ end subroutine cesm_init !=============================================================================== subroutine cesm_run() - use seq_comm_mct, only: atm_layout, lnd_layout, ice_layout, glc_layout, rof_layout, & - ocn_layout, wav_layout, esp_layout + use seq_comm_mct, only: atm_layout, lnd_layout, ice_layout, glc_layout, & + rof_layout, ocn_layout, wav_layout, esp_layout use shr_string_mod, only: shr_string_listGetIndexF ! gptl timer lookup variables @@ -2062,6 +2077,7 @@ subroutine cesm_run() ! Driver pause/resume logical :: drv_pause ! Driver writes pause restart file character(len=CL) :: drv_resume ! Driver resets state from restart file + integer :: iamroot_ESPID 101 format( A, 2i8, 12A, A, F8.2, A, F8.2 ) 102 format( A, 2i8, A, 8L3 ) @@ -2117,6 +2133,11 @@ subroutine cesm_run() ! (this is time that models should have before they return ! to the driver). Write timestamp and run alarm status !---------------------------------------------------------- + ! Note that the glcrun_avg_alarm just controls what is passed to glc in terms + ! of averaged fields - it does NOT control when glc is called currently - + ! glc will be called on the glcrun_alarm setting - but it might not be passed relevant + ! info if the time averaging period to accumulate information passed to glc is greater + ! than the glcrun interval call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod ) @@ -2127,6 +2148,7 @@ subroutine cesm_run() rofrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_rofrun) icerun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_icerun) glcrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_glcrun) + glcrun_avg_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_glcrun_avg) wavrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_wavrun) esprun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_esprun) ocnrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_ocnrun) @@ -2138,49 +2160,19 @@ subroutine cesm_run() barrier_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_barrier) pause_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_pause) - ! Determine wich components need to write pause (restart) files - if (pause_alarm) then - if (trim(pause_component_list) == 'all') then - drv_pause = .true. - call seq_infodata_putData(infodata, atm_pause=.true., & - lnd_pause=.true., ocn_pause=.true., ice_pause=.true., & - glc_pause=.true., rof_pause=.true., wav_pause=.true., & - cpl_pause=.true.) - else if (trim(pause_component_list) /= 'none') then - if (shr_string_listGetIndexF(pause_component_list, 'atm') > 0) then - call seq_infodata_putData(infodata, atm_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'lnd') > 0) then - call seq_infodata_putData(infodata, lnd_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'ocn') > 0) then - call seq_infodata_putData(infodata, ocn_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'ice') > 0) then - call seq_infodata_putData(infodata, ice_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'glc') > 0) then - call seq_infodata_putData(infodata, glc_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'rof') > 0) then - call seq_infodata_putData(infodata, rof_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'wav') > 0) then - call seq_infodata_putData(infodata, wav_pause=.true.) - end if - if ( (shr_string_listGetIndexF(pause_component_list, 'cpl') > 0) .or.& - (shr_string_listGetIndexF(pause_component_list, 'drv') > 0)) then - drv_pause = .true. - call seq_infodata_putData(infodata, cpl_pause=.true.) - end if - end if - else - drv_pause = .false. - call seq_infodata_putData(infodata, atm_pause=.false., & - lnd_pause=.false., ocn_pause=.false., ice_pause=.false., & - glc_pause=.false., rof_pause=.false., wav_pause=.false., & - cpl_pause=.false.) - end if ! pause alarm + ! Does the driver need to pause? + drv_pause = pause_alarm .and. seq_timemgr_pause_component_active(drv_index) + + ! Check alarm consistency + if (glcrun_avg_alarm .and. .not. glcrun_alarm) then + write(logunit,*) 'ERROR: glcrun_avg_alarm is true, but glcrun_alarm is false' + write(logunit,*) 'Make sure that NCPL_BASE_PERIOD, GLC_NCPL and GLC_AVG_PERIOD' + write(logunit,*) 'are set so that glc averaging only happens at glc coupling times.' + write(logunit,*) '(It is allowable for glc coupling to be more frequent than glc averaging,' + write(logunit,*) 'but not for glc averaging to be more frequent than glc coupling.)' + call shr_sys_abort(subname//' glcrun_avg_alarm is true, but glcrun_alarm is false') + end if + ! this probably belongs in seq_timemgr somewhere using proper clocks t1hr_alarm = .false. @@ -2198,8 +2190,6 @@ subroutine cesm_run() if (tod == 0) t24hr_alarm = .true. if (month==1 .and. day==1 .and. tod==0) t1yr_alarm = .true. - call seq_infodata_putData(infodata, glcrun_alarm=glcrun_alarm) - if (seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_datestop)) then if (iamroot_CPLID) then write(logunit,*) ' ' @@ -2927,20 +2917,36 @@ subroutine cesm_run() if (drv_threading) call seq_comm_setnthreads(nthreads_CPLID) if (lnd_c2_glc) then - call prep_glc_accum_avg(timer='CPL:glcprep_avg') + ! NOTE - only create appropriate input to glc if the avg_alarm is on + if (glcrun_avg_alarm) then + call prep_glc_accum_avg(timer='CPL:glcprep_avg') - ! Note that l2x_gx is obtained from mapping the module variable l2gacc_lx - call prep_glc_calc_l2x_gx(fractions_lx, timer='CPL:glcprep_lnd2glc') + ! Note that l2x_gx is obtained from mapping the module variable l2gacc_lx + call prep_glc_calc_l2x_gx(fractions_lx, timer='CPL:glcprep_lnd2glc') - call prep_glc_mrg(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') + call prep_glc_mrg(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') - call component_diag(infodata, glc, flow='x2c', comment='send glc', & - info_debug=info_debug, timer_diag='CPL:glcprep_diagav') - endif + call component_diag(infodata, glc, flow='x2c', comment='send glc', & + info_debug=info_debug, timer_diag='CPL:glcprep_diagav') + + else + call prep_glc_zero_fields() + end if ! glcrun_avg_alarm + end if ! lnd_c2_glc if (drv_threading) call seq_comm_setnthreads(nthreads_GLOID) call t_drvstopf ('CPL:GLCPREP',cplrun=.true.) - endif + + end if ! iamin_CPLID .and. glc_prognostic + + ! Set the infodata field on all tasks (not just those with iamin_CPLID). + if (glc_prognostic) then + if (glcrun_avg_alarm) then + call seq_infodata_PutData(infodata, glc_valid_input=.true.) + else + call seq_infodata_PutData(infodata, glc_valid_input=.false.) + end if + end if !---------------------------------------------------- !| cpl -> glc @@ -3726,6 +3732,10 @@ subroutine cesm_run() comp_prognostic=esp_prognostic, comp_num=comp_num_esp, & timer_barrier= 'CPL:ESP_RUN_BARRIER', timer_comp_run='CPL:ESP_RUN', & run_barriers=run_barriers, ymd=ymd, tod=tod,comp_layout=esp_layout) + !--------------------------------------------------------------------- + !| ESP computes resume options for other components -- update everyone + !--------------------------------------------------------------------- + call seq_infodata_exchange(infodata, CPLALLESPID, 'esp2cpl_run') endif !---------------------------------------------------------- @@ -3733,16 +3743,19 @@ subroutine cesm_run() !---------------------------------------------------------- call seq_infodata_GetData(infodata, cpl_resume=drv_resume) if (len_trim(drv_resume) > 0) then - if (iamroot_CPLID) then - write(logunit,103) subname,' Reading restart (resume) file ',trim(drv_resume) - call shr_sys_flush(logunit) - end if - if (iamin_CPLID) then - call seq_rest_read(drv_resume, infodata, & - atm, lnd, ice, ocn, rof, glc, wav, esp, & - fractions_ax, fractions_lx, fractions_ix, fractions_ox, & - fractions_rx, fractions_gx, fractions_wx) - end if + if (iamroot_CPLID) then + write(logunit,103) subname,' Reading restart (resume) file ',trim(drv_resume) + call shr_sys_flush(logunit) + end if + if (iamin_CPLID) then + call seq_rest_read(drv_resume, infodata, & + atm, lnd, ice, ocn, rof, glc, wav, esp, & + fractions_ax, fractions_lx, fractions_ix, fractions_ox, & + fractions_rx, fractions_gx, fractions_wx) + end if + ! Clear the resume file so we don't try to read it again + drv_resume = ' ' + call seq_infodata_PutData(infodata, cpl_resume=drv_resume) end if !---------------------------------------------------------- diff --git a/src/drivers/mct/main/prep_glc_mod.F90 b/src/drivers/mct/main/prep_glc_mod.F90 index 0bf96b83b0a8..52eeb02bfa1d 100644 --- a/src/drivers/mct/main/prep_glc_mod.F90 +++ b/src/drivers/mct/main/prep_glc_mod.F90 @@ -32,6 +32,8 @@ module prep_glc_mod public :: prep_glc_calc_l2x_gx + public :: prep_glc_zero_fields + public :: prep_glc_get_l2x_gx public :: prep_glc_get_l2gacc_lx public :: prep_glc_get_l2gacc_lx_cnt @@ -444,6 +446,31 @@ end subroutine prep_glc_map_one_field_lnd2glc !================================================================================================ + subroutine prep_glc_zero_fields() + + !--------------------------------------------------------------- + ! Description + ! Set glc inputs to zero + ! + ! This is appropriate during time intervals when we're not sending valid data to glc. + ! In principle we shouldn't need to zero the fields at these times (instead, glc + ! should just ignore the fields at these times). However, some tests (like an ERS or + ! ERI test that stops the final run segment mid-year) can fail if we don't explicitly + ! zero the fields, because these x2g fields can then differ upon restart. + + ! Local Variables + integer :: egi + type(mct_avect), pointer :: x2g_gx + !--------------------------------------------------------------- + + do egi = 1,num_inst_glc + x2g_gx => component_get_x2c_cx(glc(egi)) + call mct_aVect_zero(x2g_gx) + end do + end subroutine prep_glc_zero_fields + + !================================================================================================ + function prep_glc_get_l2x_gx() type(mct_aVect), pointer :: prep_glc_get_l2x_gx(:) prep_glc_get_l2x_gx => l2x_gx(:) diff --git a/src/drivers/mct/main/seq_io_mod.F90 b/src/drivers/mct/main/seq_io_mod.F90 index ad8649a6c110..d27fc581c6ef 100644 --- a/src/drivers/mct/main/seq_io_mod.F90 +++ b/src/drivers/mct/main/seq_io_mod.F90 @@ -37,7 +37,8 @@ module seq_io_mod use shr_kind_mod, only: r4 => shr_kind_r4, r8 => shr_kind_r8, in => shr_kind_in use shr_kind_mod, only: cl => shr_kind_cl, cs => shr_kind_cs use shr_sys_mod ! system calls - use seq_comm_mct + use seq_comm_mct, only: logunit, CPLID, seq_comm_setptrs + use seq_comm_mct, only: seq_comm_namelen, seq_comm_name use seq_flds_mod, only : seq_flds_lookup use mct_mod ! mct wrappers use pio @@ -1545,7 +1546,7 @@ subroutine seq_io_read_av(filename,gsmap,AV,dname,pre) implicit none character(len=*),intent(in) :: filename ! file type(mct_gsMap), intent(in) :: gsmap - type(mct_aVect) ,intent(inout):: AV ! data to be written + type(mct_aVect) ,intent(inout):: AV ! data to be read character(len=*),intent(in) :: dname ! name of data character(len=*),intent(in),optional :: pre ! prefix name @@ -1671,7 +1672,7 @@ subroutine seq_io_read_avs(filename,gsmap,AVS,dname,pre) implicit none character(len=*),intent(in) :: filename ! file type(mct_gsMap), intent(in) :: gsmap - type(mct_aVect) ,intent(inout):: AVS(:) ! data to be written + type(mct_aVect) ,intent(inout):: AVS(:) ! data to be read character(len=*),intent(in) :: dname ! name of data character(len=*),intent(in),optional :: pre ! prefix name diff --git a/src/drivers/mct/main/seq_rest_mod.F90 b/src/drivers/mct/main/seq_rest_mod.F90 index fa26e06d028b..6994d954b812 100644 --- a/src/drivers/mct/main/seq_rest_mod.F90 +++ b/src/drivers/mct/main/seq_rest_mod.F90 @@ -40,16 +40,17 @@ module seq_rest_mod seq_comm_iamin, CPLID, GLOID, logunit, loglevel ! "infodata" gathers various control flags into one datatype - use seq_infodata_mod + use seq_infodata_mod, only: seq_infodata_type, seq_infodata_getData ! clock & alarm routines - use seq_timemgr_mod + use seq_timemgr_mod, only: seq_timemgr_type, seq_timemgr_EClockGetData ! diagnostic routines use seq_diag_mct, only: budg_datag ! lower level io routines - use seq_io_mod + use seq_io_mod, only: seq_io_read, seq_io_write, seq_io_enddef + use seq_io_mod, only: seq_io_wopen, seq_io_close ! prep modules - coupler communication between different components use prep_ocn_mod, only: prep_ocn_get_x2oacc_ox @@ -412,7 +413,6 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & if (fk == 1) then whead = .true. wdata = .false. -!! call seq_io_redef(rest_file) elseif (fk == 2) then whead = .false. wdata = .true. diff --git a/src/drivers/mct/shr/seq_comm_mct.F90 b/src/drivers/mct/shr/seq_comm_mct.F90 index 5062704cded6..6dc12259f16f 100644 --- a/src/drivers/mct/shr/seq_comm_mct.F90 +++ b/src/drivers/mct/shr/seq_comm_mct.F90 @@ -96,7 +96,6 @@ module seq_comm_mct !!! internal communications, and one is needed for the global space. !!! All instances of a component type also share a separate communicator !!! All instances of a component type share a communicator with the coupler - !!! Note that ESP models do not need coupler communicators integer, parameter, public :: num_inst_phys = num_inst_atm + num_inst_lnd + & num_inst_ocn + num_inst_ice + & @@ -105,7 +104,7 @@ module seq_comm_mct integer, parameter, public :: num_cpl_phys = num_inst_atm + num_inst_lnd + & num_inst_ocn + num_inst_ice + & num_inst_glc + num_inst_rof + & - num_inst_wav + num_inst_wav + num_inst_esp integer, parameter :: ncomps = (1 + ncouplers + ncomptypes + nphysmod + num_inst_phys + num_cpl_phys) integer, public :: GLOID @@ -127,7 +126,7 @@ module seq_comm_mct integer, public :: CPLALLGLCID integer, public :: CPLALLROFID integer, public :: CPLALLWAVID - integer, public, parameter :: CPLALLESPID = -1 + integer, public :: CPLALLESPID integer, public :: ATMID(num_inst_atm) integer, public :: LNDID(num_inst_lnd) @@ -166,7 +165,9 @@ module seq_comm_mct integer :: gloroot ! the global task number of each comps root on all pes integer :: pethreads ! max number of threads on my task integer :: cplpe ! a common task in mpicom from the cpl group for join mpicoms + ! cplpe is used to broadcast information from the coupler to the component integer :: cmppe ! a common task in mpicom from the component group for join mpicoms + ! cmppe is used to broadcast information from the component to the coupler logical :: set ! has this datatype been set integer, pointer :: petlist(:) ! esmf pet list logical :: petlist_allocated ! whether the petlist pointer variable was allocated @@ -394,7 +395,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) num_inst_min = min(num_inst_min, num_inst_glc) num_inst_min = min(num_inst_min, num_inst_wav) num_inst_min = min(num_inst_min, num_inst_rof) - num_inst_min = min(num_inst_min, num_inst_esp) +! ESP is currently limited to one instance, should not affect other comps +! num_inst_min = min(num_inst_min, num_inst_esp) num_inst_max = num_inst_atm num_inst_max = max(num_inst_max, num_inst_lnd) num_inst_max = max(num_inst_max, num_inst_ocn) @@ -412,7 +414,10 @@ subroutine seq_comm_init(Comm_in, nmlfile) if (num_inst_glc /= num_inst_min .and. num_inst_glc /= num_inst_max) error_state = .true. if (num_inst_wav /= num_inst_min .and. num_inst_wav /= num_inst_max) error_state = .true. if (num_inst_rof /= num_inst_min .and. num_inst_rof /= num_inst_max) error_state = .true. - if (num_inst_esp /= num_inst_min .and. num_inst_esp /= num_inst_max) error_state = .true. + if (num_inst_esp /= 1) then + write(logunit,*) trim(subname),' ERROR: ESP restricted to one instance' + error_state = .true. + end if if (error_state) then write(logunit,*) trim(subname),' ERROR: num_inst inconsistent' @@ -459,6 +464,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) CPLALLROFID = count count = count + 1 CPLALLWAVID = count + count = count + 1 + CPLALLESPID = count do n = 1, num_inst_atm count = count + 1 @@ -841,6 +848,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) call seq_comm_setcomm(ESPID(n), pelist, esp_nthreads, 'ESP', n, num_inst_esp) end do call seq_comm_jcommarr(ESPID,ALLESPID,'ALLESPID',1,1) + call seq_comm_joincomm(CPLID,ALLESPID,CPLALLESPID,'CPLALLESPID',1,1) !! Count the total number of threads @@ -1156,36 +1164,6 @@ subroutine seq_comm_joincomm(ID1,ID2,ID,iname,inst,tinst) seq_comms(ID)%iamroot = .false. endif -! needs to be excluded until mpi_group_size is added to serial mpi in mct -#if (1 == 0) - if (loglevel > 3) then - ! some debug code to prove the join is working ok - ! when joining mpicomms, the local rank may be quite different - ! from either the global or local ranks of the joining comms - call mpi_group_size(seq_comms(ID1)%mpigrp,nsize,ierr) - allocate(pe_t1(nsize),pe_t2(nsize)) - do n = 1,nsize - pe_t1(n) = n-1 - pe_t2(n) = -1 - enddo - call mpi_group_translate_ranks(seq_comms(ID1)%mpigrp, nsize, pe_t1, mpigrp, pe_t2, ierr) - write(logunit,*) 'ID1 ranks ',pe_t1 - write(logunit,*) 'ID1-JOIN ranks ',pe_t2 - deallocate(pe_t1,pe_t2) - - call mpi_group_size(seq_comms(ID2)%mpigrp,nsize,ierr) - allocate(pe_t1(nsize),pe_t2(nsize)) - do n = 1,nsize - pe_t1(n) = n-1 - pe_t2(n) = -1 - enddo - call mpi_group_translate_ranks(seq_comms(ID2)%mpigrp, nsize, pe_t1, mpigrp, pe_t2, ierr) - write(logunit,*) 'ID2 ranks ',pe_t1 - write(logunit,*) 'ID2-JOIN ranks ',pe_t2 - deallocate(pe_t1,pe_t2) - endif -#endif - allocate(pe_t1(1),pe_t2(1)) pe_t1(1) = 0 call mpi_group_translate_ranks(seq_comms(ID1)%mpigrp, 1, pe_t1, mpigrp, pe_t2, ierr) diff --git a/src/drivers/mct/shr/seq_flds_mod.F90 b/src/drivers/mct/shr/seq_flds_mod.F90 index d00f84f87b19..4c9de8fd468b 100644 --- a/src/drivers/mct/shr/seq_flds_mod.F90 +++ b/src/drivers/mct/shr/seq_flds_mod.F90 @@ -121,13 +121,14 @@ module seq_flds_mod ! variables CCSM_VOC, CCSM_BGC and GLC_NEC. !==================================================================== - use shr_kind_mod, only : CX => shr_kind_CX, CXX => shr_kind_CXX - use shr_sys_mod, only : shr_sys_abort - use seq_drydep_mod, only : seq_drydep_init, seq_drydep_readnl, lnd_drydep - use seq_comm_mct, only : seq_comm_iamroot, seq_comm_setptrs, logunit - use shr_megan_mod, only : shr_megan_readnl, shr_megan_mechcomps_n - use shr_fire_emis_mod, only : shr_fire_emis_readnl, shr_fire_emis_mechcomps_n, shr_fire_emis_ztop_token - use shr_carma_mod, only : shr_carma_readnl + use shr_kind_mod , only : CX => shr_kind_CX, CXX => shr_kind_CXX + use shr_sys_mod , only : shr_sys_abort + use seq_comm_mct , only : seq_comm_iamroot, seq_comm_setptrs, logunit + use seq_drydep_mod , only : seq_drydep_init, seq_drydep_readnl, lnd_drydep + use shr_megan_mod , only : shr_megan_readnl, shr_megan_mechcomps_n + use shr_fire_emis_mod , only : shr_fire_emis_readnl, shr_fire_emis_mechcomps_n, shr_fire_emis_ztop_token + use shr_carma_mod , only : shr_carma_readnl + use shr_ndep_mod , only : shr_ndep_readnl implicit none public @@ -143,8 +144,10 @@ module seq_flds_mod character(len=CXX) :: megan_voc_fields ! List of MEGAN VOC emission fields character(len=CXX) :: fire_emis_fields ! List of fire emission fields character(len=CX) :: carma_fields ! List of CARMA fields from lnd->atm + character(len=CX) :: ndep_fields ! List of nitrogen deposition fields from atm->lnd/ocn integer :: ice_ncat ! number of sea ice thickness categories logical :: seq_flds_i2o_per_cat! .true. if select per ice thickness category fields are passed from ice to ocean + logical :: add_ndep_fields ! .true. => add ndep fields !---------------------------------------------------------------------------- ! metadata @@ -3126,7 +3129,6 @@ subroutine seq_flds_set(nmlfile, ID, infodata) units = 'm' call metadata_set(shr_fire_emis_ztop_token, longname, stdname, units) - endif !----------------------------------------------------------------------------- @@ -3146,11 +3148,31 @@ subroutine seq_flds_set(nmlfile, ID, infodata) longname = 'dry deposition velocity' stdname = 'drydep_vel' units = 'cm/sec' - call metadata_set(seq_drydep_fields, longname, stdname, units) + call metadata_set(seq_drydep_fields, longname, stdname, units) endif call seq_drydep_init( ) + !----------------------------------------------------------------------------- + ! Nitrogen Deposition fields + ! First read namelist and figure out the ndepdep field list to pass + ! Then check if file exists and if not, n_drydep will be zero + ! Then add nitrogen deposition fields to atm export, lnd import and ocn import + !----------------------------------------------------------------------------- + + call shr_ndep_readnl(nlfilename="drv_flds_in", ID=ID, ndep_fields=ndep_fields, add_ndep_fields=add_ndep_fields) + if (add_ndep_fields) then + call seq_flds_add(a2x_fluxes, ndep_fields) + call seq_flds_add(x2l_fluxes, ndep_fields) + call seq_flds_add(x2o_fluxes, ndep_fields) + + longname = 'nitrogen deposition flux' + stdname = 'nitrogen_deposition' + units = 'kg(N)/m2/sec' + + call metadata_set(ndep_fields, longname, stdname, units) + end if + !---------------------------------------------------------------------------- ! state + flux fields !---------------------------------------------------------------------------- diff --git a/src/drivers/mct/shr/seq_infodata_mod.F90 b/src/drivers/mct/shr/seq_infodata_mod.F90 index 7341e4c2af2f..d62014fa87f4 100644 --- a/src/drivers/mct/shr/seq_infodata_mod.F90 +++ b/src/drivers/mct/shr/seq_infodata_mod.F90 @@ -46,6 +46,7 @@ MODULE seq_infodata_mod ! !PUBLIC MEMBER FUNCTIONS public :: seq_infodata_Init ! Initialize + public :: seq_infodata_Init2 ! Init after clocks initialized public :: seq_infodata_GetData ! Get values from object public :: seq_infodata_PutData ! Change values public :: seq_infodata_Print ! print current info @@ -66,14 +67,6 @@ MODULE seq_infodata_mod ! Type to hold pause/resume signaling information type seq_pause_resume_type private - logical :: atm_pause = .false. ! atm write pause restart file - logical :: lnd_pause = .false. ! lnd write pause restart file - logical :: ice_pause = .false. ! ice write pause restart file - logical :: ocn_pause = .false. ! ocn write pause restart file - logical :: glc_pause = .false. ! glc write pause restart file - logical :: rof_pause = .false. ! rof write pause restart file - logical :: wav_pause = .false. ! wav write pause restart file - logical :: cpl_pause = .false. ! cpl write pause restart file character(SHR_KIND_CL) :: atm_resume(num_inst_atm) = ' ' ! atm resume file character(SHR_KIND_CL) :: lnd_resume(num_inst_lnd) = ' ' ! lnd resume file character(SHR_KIND_CL) :: ice_resume(num_inst_ice) = ' ' ! ice resume file @@ -244,12 +237,13 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: wav_phase ! wav phase integer(SHR_KIND_IN) :: esp_phase ! esp phase logical :: atm_aero ! atmosphere aerosols - logical :: glcrun_alarm ! glc run alarm logical :: glc_g2lupdate ! update glc2lnd fields in lnd model type(seq_pause_resume_type), pointer :: pause_resume => NULL() real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification + !--- set by driver and may be time varying + logical :: glc_valid_input ! is valid accumulated data being sent to prognostic glc end type seq_infodata_type ! --- public interfaces -------------------------------------------------------- @@ -293,12 +287,12 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) ! !USES: - use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit - use shr_string_mod, only : shr_string_toUpper, shr_string_listAppend - use shr_mpi_mod, only : shr_mpi_bcast - use seq_io_read_mod - use pio, only : file_desc_t - implicit none + use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit + use shr_string_mod, only : shr_string_toUpper, shr_string_listAppend + use shr_mpi_mod, only : shr_mpi_bcast + use seq_timemgr_mod, only : seq_timemgr_pause_active + use seq_io_read_mod, only : seq_io_read + use pio, only : file_desc_t ! !INPUT/OUTPUT PARAMETERS: @@ -555,6 +549,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) mct_usealltoall = .false. mct_usevector = .false. max_cplstep_time = 0.0 + !--------------------------------------------------------------------------- ! Read in namelist !--------------------------------------------------------------------------- @@ -715,15 +710,14 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) infodata%rof_phase = 1 infodata%wav_phase = 1 infodata%atm_aero = .false. - infodata%glcrun_alarm = .false. infodata%glc_g2lupdate = .false. + infodata%glc_valid_input = .true. if (associated(infodata%pause_resume)) then deallocate(infodata%pause_resume) end if nullify(infodata%pause_resume) infodata%max_cplstep_time = max_cplstep_time - !--------------------------------------------------------------- ! check orbital mode, reset unused parameters, validate settings !--------------------------------------------------------------- @@ -858,6 +852,43 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) END SUBROUTINE seq_infodata_Init +!=============================================================================== +!BOP =========================================================================== +! +! !IROUTINE: seq_infodata_Init2 -- initialize infodata structures +! +! !DESCRIPTION: +! +! Initialize infodata items that depend on the time manager setup +! +! !INTERFACE: ------------------------------------------------------------------ + +SUBROUTINE seq_infodata_Init2(infodata, ID) + +! !USES: + + use seq_timemgr_mod, only : seq_timemgr_pause_active + +! !INPUT/OUTPUT PARAMETERS: + + type(seq_infodata_type), intent(INOUT) :: infodata ! infodata object + integer(SHR_KIND_IN), intent(IN) :: ID ! seq_comm ID +!EOP + + !----- local ----- + integer :: mpicom ! MPI communicator + + call seq_comm_setptrs(ID, mpicom=mpicom) + !---------------------------------------------------------- + !| If pause/resume is active, initialize the resume data + !---------------------------------------------------------- + if (seq_timemgr_pause_active() .and. (.not. associated(infodata%pause_resume))) then + allocate(infodata%pause_resume) + end if + call seq_infodata_bcast(infodata, mpicom) + +END SUBROUTINE seq_infodata_Init2 + !=============================================================================== !=============================================================================== ! !IROUTINE: seq_infodata_GetData_explicit -- Get values from infodata object @@ -883,7 +914,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ ice_gnam, rof_gnam, glc_gnam, wav_gnam, & atm_gnam, ocn_gnam, info_debug, dead_comps, read_restart, & shr_map_dopole, vect_map, aoflux_grid, flux_epbalfact, & - nextsw_cday, precip_fact, flux_epbal, flux_albav, glcrun_alarm, & + nextsw_cday, precip_fact, flux_epbal, flux_albav, & glc_g2lupdate, atm_aero, run_barriers, esmf_map_flag, & do_budgets, do_histinit, drv_threading, flux_diurnal, gust_fac, & budget_inst, budget_daily, budget_month, wall_time_limit, & @@ -896,17 +927,16 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ cpl_cdf64, orb_iyear, orb_iyear_align, orb_mode, orb_mvelp, & orb_eccen, orb_obliqr, orb_lambm0, orb_mvelpp, wv_sat_scheme, & wv_sat_transition_start, wv_sat_use_tables, wv_sat_table_spacing, & - tfreeze_option, & + tfreeze_option, & glc_phase, rof_phase, atm_phase, lnd_phase, ocn_phase, ice_phase, & wav_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & glc_nx, glc_ny, eps_frac, eps_amask, & eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_diffmax, reprosum_recompute, & - atm_pause, lnd_pause, ocn_pause, ice_pause, glc_pause, rof_pause, & - wav_pause, cpl_pause, atm_resume, lnd_resume, ocn_resume, & - ice_resume, glc_resume, rof_resume, wav_resume, cpl_resume, & - mct_usealltoall, mct_usevector, max_cplstep_time) + atm_resume, lnd_resume, ocn_resume, ice_resume, & + glc_resume, rof_resume, wav_resume, cpl_resume, & + mct_usealltoall, mct_usevector, max_cplstep_time, glc_valid_input) implicit none @@ -1065,17 +1095,9 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(OUT) :: wav_phase ! wav phase integer(SHR_KIND_IN), optional, intent(OUT) :: esp_phase ! wav phase logical, optional, intent(OUT) :: atm_aero ! atmosphere aerosols - logical, optional, intent(OUT) :: glcrun_alarm ! glc run alarm logical, optional, intent(OUT) :: glc_g2lupdate ! update glc2lnd fields in lnd model real(shr_kind_r8), optional, intent(out) :: max_cplstep_time - logical, optional, intent(OUT) :: atm_pause ! atm write pause restart file - logical, optional, intent(OUT) :: lnd_pause ! lnd write pause restart file - logical, optional, intent(OUT) :: ice_pause ! ice write pause restart file - logical, optional, intent(OUT) :: ocn_pause ! ocn write pause restart file - logical, optional, intent(OUT) :: glc_pause ! glc write pause restart file - logical, optional, intent(OUT) :: rof_pause ! rof write pause restart file - logical, optional, intent(OUT) :: wav_pause ! wav write pause restart file - logical, optional, intent(OUT) :: cpl_pause ! cpl write pause restart file + logical, optional, intent(OUT) :: glc_valid_input character(SHR_KIND_CL), optional, intent(OUT) :: atm_resume(:) ! atm read resume state character(SHR_KIND_CL), optional, intent(OUT) :: lnd_resume(:) ! lnd read resume state character(SHR_KIND_CL), optional, intent(OUT) :: ice_resume(:) ! ice read resume state @@ -1255,64 +1277,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(wav_phase) ) wav_phase = infodata%wav_phase if ( present(esp_phase) ) esp_phase = infodata%esp_phase if ( present(atm_aero) ) atm_aero = infodata%atm_aero - if ( present(glcrun_alarm) ) glcrun_alarm = infodata%glcrun_alarm if ( present(glc_g2lupdate) ) glc_g2lupdate = infodata%glc_g2lupdate - if ( present(atm_pause) ) then - if (associated(infodata%pause_resume)) then - atm_pause = infodata%pause_resume%atm_pause - else - atm_pause = .false. - end if - end if - if ( present(lnd_pause) ) then - if (associated(infodata%pause_resume)) then - lnd_pause = infodata%pause_resume%lnd_pause - else - lnd_pause = .false. - end if - end if - if ( present(ice_pause) ) then - if (associated(infodata%pause_resume)) then - ice_pause = infodata%pause_resume%ice_pause - else - ice_pause = .false. - end if - end if - if ( present(ocn_pause) ) then - if (associated(infodata%pause_resume)) then - ocn_pause = infodata%pause_resume%ocn_pause - else - ocn_pause = .false. - end if - end if - if ( present(glc_pause) ) then - if (associated(infodata%pause_resume)) then - glc_pause = infodata%pause_resume%glc_pause - else - glc_pause = .false. - end if - end if - if ( present(rof_pause) ) then - if (associated(infodata%pause_resume)) then - rof_pause = infodata%pause_resume%rof_pause - else - rof_pause = .false. - end if - end if - if ( present(wav_pause) ) then - if (associated(infodata%pause_resume)) then - wav_pause = infodata%pause_resume%wav_pause - else - wav_pause = .false. - end if - end if - if ( present(cpl_pause) ) then - if (associated(infodata%pause_resume)) then - cpl_pause = infodata%pause_resume%cpl_pause - else - cpl_pause = .false. - end if - end if if ( present(atm_resume) ) then if (associated(infodata%pause_resume)) then atm_resume(:) = infodata%pause_resume%atm_resume(:) @@ -1370,6 +1335,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ end if end if if ( present(max_cplstep_time) ) max_cplstep_time = infodata%max_cplstep_time + if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input END SUBROUTINE seq_infodata_GetData_explicit @@ -1385,7 +1351,7 @@ END SUBROUTINE seq_infodata_GetData_explicit SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & comp_present, comp_prognostic, comp_gnam, histavg_comp, & - comp_phase, comp_nx, comp_ny, comp_pause, comp_resume) + comp_phase, comp_nx, comp_ny, comp_resume) implicit none @@ -1401,7 +1367,6 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & integer(SHR_KIND_IN), optional, intent(OUT) :: comp_ny ! nx,ny 2d grid size global integer(SHR_KIND_IN), optional, intent(OUT) :: comp_phase logical, optional, intent(OUT) :: histavg_comp - logical, optional, intent(OUT) :: comp_pause character(SHR_KIND_CL), optional, intent(OUT) :: comp_resume(:) !----- local ----- @@ -1413,37 +1378,37 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & call seq_infodata_GetData(infodata, atm_present=comp_present, & atm_prognostic=comp_prognostic, atm_gnam=comp_gnam, & atm_phase=comp_phase, atm_nx=comp_nx, atm_ny=comp_ny, & - histavg_atm=histavg_comp, atm_pause=comp_pause, atm_resume=comp_resume) + histavg_atm=histavg_comp, atm_resume=comp_resume) else if (component_firstletter == 'l') then call seq_infodata_GetData(infodata, lnd_present=comp_present, & lnd_prognostic=comp_prognostic, lnd_gnam=comp_gnam, & lnd_phase=comp_phase, lnd_nx=comp_nx, lnd_ny=comp_ny, & - histavg_lnd=histavg_comp, lnd_pause=comp_pause, lnd_resume=comp_resume) + histavg_lnd=histavg_comp, lnd_resume=comp_resume) else if (component_firstletter == 'i') then call seq_infodata_GetData(infodata, ice_present=comp_present, & ice_prognostic=comp_prognostic, ice_gnam=comp_gnam, & ice_phase=comp_phase, ice_nx=comp_nx, ice_ny=comp_ny, & - histavg_ice=histavg_comp, ice_pause=comp_pause, ice_resume=comp_resume) + histavg_ice=histavg_comp, ice_resume=comp_resume) else if (component_firstletter == 'o') then call seq_infodata_GetData(infodata, ocn_present=comp_present, & ocn_prognostic=comp_prognostic, ocn_gnam=comp_gnam, & ocn_phase=comp_phase, ocn_nx=comp_nx, ocn_ny=comp_ny, & - histavg_ocn=histavg_comp, ocn_pause=comp_pause, ocn_resume=comp_resume) + histavg_ocn=histavg_comp, ocn_resume=comp_resume) else if (component_firstletter == 'r') then call seq_infodata_GetData(infodata, rof_present=comp_present, & rof_prognostic=comp_prognostic, rof_gnam=comp_gnam, & rof_phase=comp_phase, rof_nx=comp_nx, rof_ny=comp_ny, & - histavg_rof=histavg_comp, rof_pause=comp_pause, rof_resume=comp_resume) + histavg_rof=histavg_comp, rof_resume=comp_resume) else if (component_firstletter == 'g') then call seq_infodata_GetData(infodata, glc_present=comp_present, & glc_prognostic=comp_prognostic, glc_gnam=comp_gnam, & glc_phase=comp_phase, glc_nx=comp_nx, glc_ny=comp_ny, & - histavg_glc=histavg_comp, glc_pause=comp_pause, glc_resume=comp_resume) + histavg_glc=histavg_comp, glc_resume=comp_resume) else if (component_firstletter == 'w') then call seq_infodata_GetData(infodata, wav_present=comp_present, & wav_prognostic=comp_prognostic, wav_gnam=comp_gnam, & wav_phase=comp_phase, wav_nx=comp_nx, wav_ny=comp_ny, & - histavg_wav=histavg_comp, wav_pause=comp_pause, wav_resume=comp_resume) + histavg_wav=histavg_comp, wav_resume=comp_resume) else if (component_firstletter == 'e') then if (present(comp_gnam)) then comp_gnam = '' @@ -1469,12 +1434,6 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & write(logunit,*) trim(subname),' Note: ESP type has no histavg property' end if end if - if (present(comp_pause)) then - comp_pause = .false. - if ((loglevel > 1) .and. seq_comm_iamroot(1)) then - write(logunit,*) trim(subname),' Note: ESP type has no pause property' - end if - end if if (present(comp_resume)) then comp_resume = ' ' if ((loglevel > 1) .and. seq_comm_iamroot(1)) then @@ -1515,7 +1474,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ ice_gnam, rof_gnam, glc_gnam, wav_gnam, & atm_gnam, ocn_gnam, info_debug, dead_comps, read_restart, & shr_map_dopole, vect_map, aoflux_grid, run_barriers, & - nextsw_cday, precip_fact, flux_epbal, flux_albav, glcrun_alarm, & + nextsw_cday, precip_fact, flux_epbal, flux_albav, & glc_g2lupdate, atm_aero, esmf_map_flag, wall_time_limit, & do_budgets, do_histinit, drv_threading, flux_diurnal, gust_fac, & budget_inst, budget_daily, budget_month, force_stop_at, & @@ -1535,10 +1494,9 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ glc_nx, glc_ny, eps_frac, eps_amask, & eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_diffmax, reprosum_recompute, & - atm_pause, lnd_pause, ocn_pause, ice_pause, glc_pause, rof_pause, & - wav_pause, cpl_pause, atm_resume, lnd_resume, ocn_resume, & - ice_resume, glc_resume, rof_resume, wav_resume, cpl_resume, & - mct_usealltoall, mct_usevector ) + atm_resume, lnd_resume, ocn_resume, ice_resume, & + glc_resume, rof_resume, wav_resume, cpl_resume, & + mct_usealltoall, mct_usevector, glc_valid_input) implicit none @@ -1696,16 +1654,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(IN) :: wav_phase ! wav phase integer(SHR_KIND_IN), optional, intent(IN) :: esp_phase ! esp phase logical, optional, intent(IN) :: atm_aero ! atm aerosols - logical, optional, intent(IN) :: glcrun_alarm ! glc run alarm logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model - logical, optional, intent(IN) :: atm_pause ! atm pause - logical, optional, intent(IN) :: lnd_pause ! lnd pause - logical, optional, intent(IN) :: ice_pause ! ice pause - logical, optional, intent(IN) :: ocn_pause ! ocn pause - logical, optional, intent(IN) :: glc_pause ! glc pause - logical, optional, intent(IN) :: rof_pause ! rof pause - logical, optional, intent(IN) :: wav_pause ! wav pause - logical, optional, intent(IN) :: cpl_pause ! cpl pause + logical, optional, intent(IN) :: glc_valid_input character(SHR_KIND_CL), optional, intent(IN) :: atm_resume(:) ! atm resume character(SHR_KIND_CL), optional, intent(IN) :: lnd_resume(:) ! lnd resume character(SHR_KIND_CL), optional, intent(IN) :: ice_resume(:) ! ice resume @@ -1873,72 +1823,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(wav_phase) ) infodata%wav_phase = wav_phase if ( present(esp_phase) ) infodata%esp_phase = esp_phase if ( present(atm_aero) ) infodata%atm_aero = atm_aero - if ( present(glcrun_alarm) ) infodata%glcrun_alarm = glcrun_alarm if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate - if ( present(atm_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%atm_pause = atm_pause - else if (atm_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%atm_pause = atm_pause - end if - end if - if ( present(lnd_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%lnd_pause = lnd_pause - else if (lnd_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%lnd_pause = lnd_pause - end if - end if - if ( present(ice_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%ice_pause = ice_pause - else if (ice_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%ice_pause = ice_pause - end if - end if - if ( present(ocn_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%ocn_pause = ocn_pause - else if (ocn_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%ocn_pause = ocn_pause - end if - end if - if ( present(glc_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%glc_pause = glc_pause - else if (glc_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%glc_pause = glc_pause - end if - end if - if ( present(rof_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%rof_pause = rof_pause - else if (rof_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%rof_pause = rof_pause - end if - end if - if ( present(wav_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%wav_pause = wav_pause - else if (wav_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%wav_pause = wav_pause - end if - end if - if ( present(cpl_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%cpl_pause = cpl_pause - else if (cpl_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%cpl_pause = cpl_pause - end if - end if + if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input if ( present(atm_resume) ) then if (associated(infodata%pause_resume)) then infodata%pause_resume%atm_resume(:) = atm_resume(:) @@ -2018,7 +1904,7 @@ END SUBROUTINE seq_infodata_PutData_explicit SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & comp_present, comp_prognostic, comp_gnam, & - histavg_comp, comp_phase, comp_nx, comp_ny, comp_pause, comp_resume) + histavg_comp, comp_phase, comp_nx, comp_ny, comp_resume) implicit none @@ -2033,7 +1919,6 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & integer(SHR_KIND_IN), optional, intent(IN) :: comp_ny ! nx,ny 2d grid size global integer(SHR_KIND_IN), optional, intent(IN) :: comp_phase logical, optional, intent(IN) :: histavg_comp - logical, optional, intent(IN) :: comp_pause character(SHR_KIND_CL), optional, intent(IN) :: comp_resume(:) !EOP @@ -2047,37 +1932,37 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & call seq_infodata_PutData(infodata, atm_present=comp_present, & atm_prognostic=comp_prognostic, atm_gnam=comp_gnam, & atm_phase=comp_phase, atm_nx=comp_nx, atm_ny=comp_ny, & - histavg_atm=histavg_comp, atm_pause=comp_pause, atm_resume=comp_resume) + histavg_atm=histavg_comp, atm_resume=comp_resume) else if (component_firstletter == 'l') then call seq_infodata_PutData(infodata, lnd_present=comp_present, & lnd_prognostic=comp_prognostic, lnd_gnam=comp_gnam, & lnd_phase=comp_phase, lnd_nx=comp_nx, lnd_ny=comp_ny, & - histavg_lnd=histavg_comp, lnd_pause=comp_pause, lnd_resume=comp_resume) + histavg_lnd=histavg_comp, lnd_resume=comp_resume) else if (component_firstletter == 'i') then call seq_infodata_PutData(infodata, ice_present=comp_present, & ice_prognostic=comp_prognostic, ice_gnam=comp_gnam, & ice_phase=comp_phase, ice_nx=comp_nx, ice_ny=comp_ny, & - histavg_ice=histavg_comp, ice_pause=comp_pause, ice_resume=comp_resume) + histavg_ice=histavg_comp, ice_resume=comp_resume) else if (component_firstletter == 'o') then call seq_infodata_PutData(infodata, ocn_present=comp_present, & ocn_prognostic=comp_prognostic, ocn_gnam=comp_gnam, & ocn_phase=comp_phase, ocn_nx=comp_nx, ocn_ny=comp_ny, & - histavg_ocn=histavg_comp, ocn_pause=comp_pause, ocn_resume=comp_resume) + histavg_ocn=histavg_comp, ocn_resume=comp_resume) else if (component_firstletter == 'r') then call seq_infodata_PutData(infodata, rof_present=comp_present, & rof_prognostic=comp_prognostic, rof_gnam=comp_gnam, & rof_phase=comp_phase, rof_nx=comp_nx, rof_ny=comp_ny, & - histavg_rof=histavg_comp, rof_pause=comp_pause, rof_resume=comp_resume) + histavg_rof=histavg_comp, rof_resume=comp_resume) else if (component_firstletter == 'g') then call seq_infodata_PutData(infodata, glc_present=comp_present, & glc_prognostic=comp_prognostic, glc_gnam=comp_gnam, & glc_phase=comp_phase, glc_nx=comp_nx, glc_ny=comp_ny, & - histavg_glc=histavg_comp, glc_pause=comp_pause, glc_resume=comp_resume) + histavg_glc=histavg_comp, glc_resume=comp_resume) else if (component_firstletter == 'w') then call seq_infodata_PutData(infodata, wav_present=comp_present, & wav_prognostic=comp_prognostic, wav_gnam=comp_gnam, & wav_phase=comp_phase, wav_nx=comp_nx, wav_ny=comp_ny, & - histavg_wav=histavg_comp, wav_pause=comp_pause, wav_resume=comp_resume) + histavg_wav=histavg_comp, wav_resume=comp_resume) else if (component_firstletter == 'e') then if ((loglevel > 1) .and. seq_comm_iamroot(1)) then if (present(comp_gnam)) then @@ -2092,9 +1977,6 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & if (present(histavg_comp)) then write(logunit,*) trim(subname),' Note: ESP type has no histavg property' end if - if (present(comp_pause)) then - write(logunit,*) trim(subname),' Note: ESP type has no pause property' - end if if (present(comp_resume)) then write(logunit,*) trim(subname),' Note: ESP type has no resume property' end if @@ -2111,6 +1993,74 @@ END SUBROUTINE seq_infodata_PutData_bytype #endif ! ^ ifndef CPRPGI +!=============================================================================== +!BOP =========================================================================== +! +! !IROUTINE: seq_infodata_pauseresume_bcast -- Broadcast pause/resume data from root pe +! +! !DESCRIPTION: +! +! Broadcast the pause_resume data from an infodata across pes +! +! !INTERFACE: ------------------------------------------------------------------ + +subroutine seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast) + + use shr_mpi_mod, only : shr_mpi_bcast + +! !INPUT/OUTPUT PARAMETERS: + + type(seq_infodata_type), intent(INOUT) :: infodata ! assume valid on root pe + integer(SHR_KIND_IN), intent(IN) :: mpicom ! MPI Communicator + integer(SHR_KIND_IN), optional, intent(IN) :: pebcast ! pe sending + +!EOP + + !----- local ----- + integer :: ind + integer(SHR_KIND_IN) :: pebcast_local + character(len=*), parameter :: subname = '(seq_infodata_pauseresume_bcast) ' + + if (present(pebcast)) then + pebcast_local = pebcast + else + pebcast_local = 0 + end if + + if (associated(infodata%pause_resume)) then + do ind = 1, num_inst_atm + call shr_mpi_bcast(infodata%pause_resume%atm_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_lnd + call shr_mpi_bcast(infodata%pause_resume%lnd_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_ice + call shr_mpi_bcast(infodata%pause_resume%ice_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_ocn + call shr_mpi_bcast(infodata%pause_resume%ocn_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_glc + call shr_mpi_bcast(infodata%pause_resume%glc_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_rof + call shr_mpi_bcast(infodata%pause_resume%rof_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_wav + call shr_mpi_bcast(infodata%pause_resume%wav_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + call shr_mpi_bcast(infodata%pause_resume%cpl_resume, mpicom, & + pebcast=pebcast_local) + end if +end subroutine seq_infodata_pauseresume_bcast + !=============================================================================== !BOP =========================================================================== ! @@ -2291,40 +2241,10 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%rof_phase, mpicom) call shr_mpi_bcast(infodata%wav_phase, mpicom) call shr_mpi_bcast(infodata%atm_aero, mpicom) - call shr_mpi_bcast(infodata%glcrun_alarm, mpicom) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom) - if (associated(infodata%pause_resume)) then - call shr_mpi_bcast(infodata%pause_resume%atm_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%lnd_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%ice_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%ocn_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%glc_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%rof_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%wav_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%cpl_pause, mpicom) - do ind = 1, num_inst_atm - call shr_mpi_bcast(infodata%pause_resume%atm_resume(ind), mpicom) - end do - do ind = 1, num_inst_lnd - call shr_mpi_bcast(infodata%pause_resume%lnd_resume(ind), mpicom) - end do - do ind = 1, num_inst_ice - call shr_mpi_bcast(infodata%pause_resume%ice_resume(ind), mpicom) - end do - do ind = 1, num_inst_ocn - call shr_mpi_bcast(infodata%pause_resume%ocn_resume(ind), mpicom) - end do - do ind = 1, num_inst_glc - call shr_mpi_bcast(infodata%pause_resume%glc_resume(ind), mpicom) - end do - do ind = 1, num_inst_rof - call shr_mpi_bcast(infodata%pause_resume%rof_resume(ind), mpicom) - end do - do ind = 1, num_inst_wav - call shr_mpi_bcast(infodata%pause_resume%wav_resume(ind), mpicom) - end do - call shr_mpi_bcast(infodata%pause_resume%cpl_resume, mpicom) - end if + call shr_mpi_bcast(infodata%glc_valid_input, mpicom) + + call seq_infodata_pauseresume_bcast(infodata, mpicom) end subroutine seq_infodata_bcast @@ -2355,7 +2275,8 @@ subroutine seq_infodata_Exchange(infodata,ID,type) !----- local ----- integer(SHR_KIND_IN) :: mpicom ! mpicom - integer(SHR_KIND_IN) :: pebcast ! pe sending + integer(SHR_KIND_IN) :: cmppe ! component 'root' for broadcast + integer(SHR_KIND_IN) :: cplpe ! coupler 'root' for broadcast logical :: atm2cpli,atm2cplr logical :: lnd2cpli,lnd2cplr logical :: rof2cpli,rof2cplr @@ -2363,6 +2284,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) logical :: ice2cpli,ice2cplr logical :: glc2cpli,glc2cplr logical :: wav2cpli,wav2cplr + logical :: esp2cpli,esp2cplr logical :: cpl2i,cpl2r logical :: logset logical :: deads ! local variable to hold info temporarily @@ -2372,8 +2294,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) ! Notes: !------------------------------------------------------------------------------- - ! assume the comp pe is going to broadcast, change to cplpe below if appropriate - call seq_comm_setptrs(ID,mpicom=mpicom,cmppe=pebcast) + call seq_comm_setptrs(ID, mpicom=mpicom, cmppe=cmppe, cplpe=cplpe) logset = .false. @@ -2391,6 +2312,8 @@ subroutine seq_infodata_Exchange(infodata,ID,type) glc2cplr = .false. wav2cpli = .false. wav2cplr = .false. + esp2cpli = .false. + esp2cplr = .false. cpl2i = .false. cpl2r = .false. @@ -2466,16 +2389,26 @@ subroutine seq_infodata_Exchange(infodata,ID,type) logset = .true. endif + if (trim(type) == 'esp2cpl_init') then + esp2cpli = .true. + esp2cplr = .true. + logset = .true. + endif + if (trim(type) == 'esp2cpl_run') then + esp2cplr = .true. + logset = .true. + endif + if (trim(type) == 'cpl2atm_init' .or. & trim(type) == 'cpl2lnd_init' .or. & trim(type) == 'cpl2rof_init' .or. & trim(type) == 'cpl2ocn_init' .or. & trim(type) == 'cpl2glc_init' .or. & trim(type) == 'cpl2wav_init' .or. & + trim(type) == 'cpl2esp_init' .or. & trim(type) == 'cpl2ice_init') then cpl2i = .true. cpl2r = .true. - call seq_comm_setptrs(ID,cplpe=pebcast) logset = .true. endif @@ -2487,7 +2420,6 @@ subroutine seq_infodata_Exchange(infodata,ID,type) trim(type) == 'cpl2wav_run' .or. & trim(type) == 'cpl2ice_run') then cpl2r = .true. - call seq_comm_setptrs(ID,cplpe=pebcast) logset = .true. endif @@ -2501,129 +2433,143 @@ subroutine seq_infodata_Exchange(infodata,ID,type) ! --- now execute exchange --- if (atm2cpli) then - call shr_mpi_bcast(infodata%atm_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_ny, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_aero, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%atm_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_aero, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (lnd2cpli) then - call shr_mpi_bcast(infodata%lnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%lnd_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (rof2cpli) then - call shr_mpi_bcast(infodata%rof_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rofice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_ny, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%flood_present, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%rof_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rofice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%flood_present, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (ocn2cpli) then - call shr_mpi_bcast(infodata%ocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocnrof_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%ocn_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (ice2cpli) then - call shr_mpi_bcast(infodata%ice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%iceberg_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (glc2cpli) then - call shr_mpi_bcast(infodata%glc_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glclnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%glc_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glclnd_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glcocn_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glcice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (wav2cpli) then - call shr_mpi_bcast(infodata%wav_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%wav_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif + if (esp2cpli) then + call shr_mpi_bcast(infodata%esp_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%esp_prognostic, mpicom, pebcast=cmppe) + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cmppe) + endif + if (cpl2i) then - call shr_mpi_bcast(infodata%atm_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rofice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%flood_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocnrof_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%iceberg_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glclnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%dead_comps, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_aero, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%atm_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%atm_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%lnd_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%lnd_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rof_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rofice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rof_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%flood_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glclnd_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glcocn_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glcice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%wav_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%wav_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%esp_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%esp_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%dead_comps, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%atm_aero, mpicom, pebcast=cplpe) endif + ! Run-time data exchanges if (atm2cplr) then - call shr_mpi_bcast(infodata%nextsw_cday, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%nextsw_cday, mpicom, pebcast=cmppe) endif if (ocn2cplr) then - call shr_mpi_bcast(infodata%precip_fact, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + endif + + if (esp2cplr) then + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cmppe) endif if (cpl2r) then - call shr_mpi_bcast(infodata%nextsw_cday, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%precip_fact, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcrun_alarm, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%nextsw_cday, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2754,33 +2700,6 @@ subroutine seq_infodata_Check( infodata ) call shr_sys_abort(subname//': vect_map invalid = '//trim(infodata%vect_map)) endif - if (associated(infodata%pause_resume)) then - if (infodata%pause_resume%atm_pause .and. ANY(len_trim(infodata%pause_resume%atm_resume) > 0)) then - call shr_sys_abort(subname//': If atm_pause is .true., then atm_resume should not be set') - end if - if (infodata%pause_resume%lnd_pause .and. ANY(len_trim(infodata%pause_resume%lnd_resume) > 0)) then - call shr_sys_abort(subname//': If lnd_pause is .true., then lnd_resume should not be set') - end if - if (infodata%pause_resume%ocn_pause .and. ANY(len_trim(infodata%pause_resume%ocn_resume) > 0)) then - call shr_sys_abort(subname//': If ocn_pause is .true., then ocn_resume should not be set') - end if - if (infodata%pause_resume%ice_pause .and. ANY(len_trim(infodata%pause_resume%ice_resume) > 0)) then - call shr_sys_abort(subname//': If ice_pause is .true., then ice_resume should not be set') - end if - if (infodata%pause_resume%rof_pause .and. ANY(len_trim(infodata%pause_resume%rof_resume) > 0)) then - call shr_sys_abort(subname//': If rof_pause is .true., then rof_resume should not be set') - end if - if (infodata%pause_resume%glc_pause .and. ANY(len_trim(infodata%pause_resume%glc_resume) > 0)) then - call shr_sys_abort(subname//': If glc_pause is .true., then glc_resume should not be set') - end if - if (infodata%pause_resume%wav_pause .and. ANY(len_trim(infodata%pause_resume%wav_resume) > 0)) then - call shr_sys_abort(subname//': If wav_pause is .true., then wav_resume should not be set') - end if - if (infodata%pause_resume%cpl_pause .and. (len_trim(infodata%pause_resume%cpl_resume) > 0)) then - call shr_sys_abort(subname//': If cpl_pause is .true., then cpl_resume should not be set') - end if - end if - END SUBROUTINE seq_infodata_Check !=============================================================================== @@ -2992,17 +2911,8 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'rof_phase = ', infodata%rof_phase write(logunit,F0S) subname,'wav_phase = ', infodata%wav_phase - write(logunit,F0L) subname,'glcrun_alarm = ', infodata%glcrun_alarm write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate if (associated(infodata%pause_resume)) then - write(logunit,F0L) subname,'atm_pause = ', infodata%pause_resume%atm_pause - write(logunit,F0L) subname,'lnd_pause = ', infodata%pause_resume%lnd_pause - write(logunit,F0L) subname,'ocn_pause = ', infodata%pause_resume%ocn_pause - write(logunit,F0L) subname,'ice_pause = ', infodata%pause_resume%ice_pause - write(logunit,F0L) subname,'glc_pause = ', infodata%pause_resume%glc_pause - write(logunit,F0S) subname,'rof_pause = ', infodata%pause_resume%rof_pause - write(logunit,F0L) subname,'wav_pause = ', infodata%pause_resume%wav_pause - write(logunit,F0L) subname,'cpl_pause = ', infodata%pause_resume%cpl_pause do ind = 1, num_inst_atm if (len_trim(infodata%pause_resume%atm_resume(ind)) > 0) then write(logunit,FIA) subname,'atm_resume(',ind,') = ', trim(infodata%pause_resume%atm_resume(ind)) diff --git a/src/drivers/mct/shr/seq_timemgr_mod.F90 b/src/drivers/mct/shr/seq_timemgr_mod.F90 index 8a297ce9e048..ed36dbdfd2a8 100644 --- a/src/drivers/mct/shr/seq_timemgr_mod.F90 +++ b/src/drivers/mct/shr/seq_timemgr_mod.F90 @@ -1,8 +1,4 @@ !=============================================================================== -! SVN $Id: seq_timemgr_mod.F90 68253 2015-02-18 22:24:57Z mvertens $ -! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/drv/seq_mct/trunk_tags/drvseq5_1_15/shr/seq_timemgr_mod.F90 $ -!=============================================================================== -!BOP =========================================================================== ! ! !MODULE: seq_timemgr_mod --- Time-manager module ! @@ -60,9 +56,11 @@ module seq_timemgr_mod public :: seq_timemgr_restartAlarmIsOn ! Is a restart alarm ringing public :: seq_timemgr_stopAlarmIsOn ! Is a stop alarm ringing public :: seq_timemgr_historyAlarmIsOn ! Is a history alarm ringing - - ! --- Data that belongs to the driver (but is here to avoid loops) - character(SHR_KIND_CS),public :: seq_timemgr_pause_component_list ! Pause - resume components + public :: seq_timemgr_pauseAlarmIsOn ! Is a pause alarm ringing + ! --- ESP components need to know about the state of other components + public :: seq_timemgr_pause_active ! Pause/resume is enabled + public :: seq_timemgr_pause_component_index ! Index of named component + public :: seq_timemgr_pause_component_active ! .true. is comp should pause ! ! PUBLIC PARAMETERS: @@ -104,6 +102,7 @@ module seq_timemgr_mod ! seq_timemgr_alarm_ocnrun ! seq_timemgr_alarm_icerun ! seq_timemgr_alarm_glcrun +! seq_timemgr_alarm_glcrun_avg ! seq_timemgr_alarm_wavrun ! seq_timemgr_alarm_esprun ! seq_timemgr_alarm_ocnnext @@ -112,8 +111,6 @@ module seq_timemgr_mod ! seq_timemgr_alarm_pause ! seq_timemgr_alarm_barrier -!EOP - private:: seq_timemgr_alarmGet private:: seq_timemgr_alarmInit private:: seq_timemgr_EClockInit @@ -141,7 +138,8 @@ module seq_timemgr_mod seq_timemgr_optYearly = "yearly" , & seq_timemgr_optDate = "date" , & seq_timemgr_optIfdays0 = "ifdays0" , & - seq_timemgr_optEnd = "end" + seq_timemgr_optEnd = "end" , & + seq_timemgr_optGLCCouplingPeriod = "glc_coupling_period" integer(SHR_KIND_IN),private,parameter :: & seq_timemgr_nclock_drv = 1, & @@ -153,6 +151,7 @@ module seq_timemgr_mod seq_timemgr_nclock_wav = 7, & seq_timemgr_nclock_rof = 8, & seq_timemgr_nclock_esp = 9 + integer(SHR_KIND_IN),private,parameter :: max_clocks = 9 character(len=*),public,parameter :: & seq_timemgr_clock_drv = 'seq_timemgr_clock_drv' , & @@ -168,68 +167,77 @@ module seq_timemgr_mod (/'drv ','atm ','lnd ','ocn ', & 'ice ','glc ','wav ','rof ','esp '/) + ! Alarms on both component clocks and driver clock integer(SHR_KIND_IN),private,parameter :: & - seq_timemgr_nalarm_restart = 1, & - seq_timemgr_nalarm_run = 2, & - seq_timemgr_nalarm_stop = 3, & - seq_timemgr_nalarm_datestop= 4, & - seq_timemgr_nalarm_history = 5, & - seq_timemgr_nalarm_atmrun = 6, & - seq_timemgr_nalarm_lndrun = 7, & - seq_timemgr_nalarm_ocnrun = 8, & - seq_timemgr_nalarm_icerun = 9, & - seq_timemgr_nalarm_glcrun =10, & - seq_timemgr_nalarm_ocnnext =11, & - seq_timemgr_nalarm_tprof =12, & - seq_timemgr_nalarm_histavg =13, & - seq_timemgr_nalarm_rofrun =14, & - seq_timemgr_nalarm_wavrun =15, & - seq_timemgr_nalarm_esprun =16, & - seq_timemgr_nalarm_pause =17, & - seq_timemgr_nalarm_barrier =18, & + seq_timemgr_nalarm_restart = 1 , & ! driver and component clock alarm + seq_timemgr_nalarm_run = 2 , & ! driver and component clock alarm + seq_timemgr_nalarm_stop = 3 , & ! driver and component clock alarm + seq_timemgr_nalarm_datestop = 4 , & ! driver and component clock alarm + seq_timemgr_nalarm_history = 5 , & ! driver and component clock alarm + seq_timemgr_nalarm_atmrun = 6 , & ! driver only clock alarm + seq_timemgr_nalarm_lndrun = 7 , & ! driver only clock alarm + seq_timemgr_nalarm_ocnrun = 8 , & ! driver only clock alarm + seq_timemgr_nalarm_icerun = 9 , & ! driver only clock alarm + seq_timemgr_nalarm_glcrun =10 , & ! driver only clock alarm + seq_timemgr_nalarm_glcrun_avg =11 , & ! driver only clock alarm + seq_timemgr_nalarm_ocnnext =12 , & ! driver only clock alarm + seq_timemgr_nalarm_tprof =13 , & ! driver and component clock alarm + seq_timemgr_nalarm_histavg =14 , & ! driver and component clock alarm + seq_timemgr_nalarm_rofrun =15 , & ! driver only clock alarm + seq_timemgr_nalarm_wavrun =16 , & ! driver only clock alarm + seq_timemgr_nalarm_esprun =17 , & ! driver only clock alarm + seq_timemgr_nalarm_pause =18 , & + seq_timemgr_nalarm_barrier =19 , & ! driver and component clock alarm max_alarms = seq_timemgr_nalarm_barrier + character(len=*),public,parameter :: & - seq_timemgr_alarm_restart = 'seq_timemgr_alarm_restart ', & - seq_timemgr_alarm_run = 'seq_timemgr_alarm_run ', & - seq_timemgr_alarm_stop = 'seq_timemgr_alarm_stop ', & - seq_timemgr_alarm_datestop= 'seq_timemgr_alarm_datestop', & - seq_timemgr_alarm_history = 'seq_timemgr_alarm_history ', & - seq_timemgr_alarm_atmrun = 'seq_timemgr_alarm_atmrun ', & - seq_timemgr_alarm_lndrun = 'seq_timemgr_alarm_lndrun ', & - seq_timemgr_alarm_ocnrun = 'seq_timemgr_alarm_ocnrun ', & - seq_timemgr_alarm_icerun = 'seq_timemgr_alarm_icerun ', & - seq_timemgr_alarm_glcrun = 'seq_timemgr_alarm_glcrun ', & - seq_timemgr_alarm_ocnnext = 'seq_timemgr_alarm_ocnnext ', & - seq_timemgr_alarm_tprof = 'seq_timemgr_alarm_tprof ', & - seq_timemgr_alarm_histavg = 'seq_timemgr_alarm_histavg ', & - seq_timemgr_alarm_rofrun = 'seq_timemgr_alarm_rofrun ', & - seq_timemgr_alarm_wavrun = 'seq_timemgr_alarm_wavrun ', & - seq_timemgr_alarm_esprun = 'seq_timemgr_alarm_esprun ', & - seq_timemgr_alarm_pause = 'seq_timemgr_alarm_pause ', & - seq_timemgr_alarm_barrier = 'seq_timemgr_alarm_barrier ' + seq_timemgr_alarm_restart = 'seq_timemgr_alarm_restart ', & + seq_timemgr_alarm_run = 'seq_timemgr_alarm_run ', & + seq_timemgr_alarm_stop = 'seq_timemgr_alarm_stop ', & + seq_timemgr_alarm_datestop = 'seq_timemgr_alarm_datestop', & + seq_timemgr_alarm_history = 'seq_timemgr_alarm_history ', & + seq_timemgr_alarm_atmrun = 'seq_timemgr_alarm_atmrun ', & + seq_timemgr_alarm_lndrun = 'seq_timemgr_alarm_lndrun ', & + seq_timemgr_alarm_ocnrun = 'seq_timemgr_alarm_ocnrun ', & + seq_timemgr_alarm_icerun = 'seq_timemgr_alarm_icerun ', & + seq_timemgr_alarm_glcrun = 'seq_timemgr_alarm_glcrun ', & + seq_timemgr_alarm_glcrun_avg = 'seq_timemgr_alarm_glcrun_avg' , & + seq_timemgr_alarm_ocnnext = 'seq_timemgr_alarm_ocnnext ', & + seq_timemgr_alarm_tprof = 'seq_timemgr_alarm_tprof ', & + seq_timemgr_alarm_histavg = 'seq_timemgr_alarm_histavg ', & + seq_timemgr_alarm_rofrun = 'seq_timemgr_alarm_rofrun ', & + seq_timemgr_alarm_wavrun = 'seq_timemgr_alarm_wavrun ', & + seq_timemgr_alarm_esprun = 'seq_timemgr_alarm_esprun ', & + seq_timemgr_alarm_pause = 'seq_timemgr_alarm_pause ', & + seq_timemgr_alarm_barrier = 'seq_timemgr_alarm_barrier ' + + ! Active pause - resume components + logical, private :: pause_active(max_clocks) = .false. type EClock_pointer ! needed for array of pointers type(ESMF_Clock),pointer :: EClock => null() end type EClock_pointer + type seq_timemgr_type private type(EClock_pointer) :: ECP(max_clocks) ! ESMF clocks, array of pointers type(ESMF_Alarm) :: EAlarm(max_clocks,max_alarms) ! array of clock alarms end type seq_timemgr_type - ! --- Private local data ------------------------------------------------------- + ! --- Private local data ---------------------------------------------------- + + type(ESMF_Calendar), target, save :: seq_timemgr_cal ! calendar + character(SHR_KIND_CL) ,save :: seq_timemgr_calendar ! calendar string + integer, parameter :: SecPerDay = 86400 ! Seconds per day - type(ESMF_Calendar), target, save :: seq_timemgr_cal ! calendar - character(SHR_KIND_CL) ,save :: seq_timemgr_calendar ! calendar string - logical :: seq_timemgr_end_restart ! write restarts at end of run? - logical, save :: seq_timemgr_setCalendar = .false. ! if calendar has been set - integer, parameter :: SecPerDay = 86400 ! Seconds per day + integer :: seq_timemgr_pause_sig_index ! Index of pause comp with smallest dt + logical :: seq_timemgr_esp_run_on_pause ! Run ESP component on pause cycle + logical :: seq_timemgr_end_restart ! write restarts at end of run? !=============================================================================== contains -!=============================================================================== !=============================================================================== ! !IROUTINE: seq_timemgr_clockInit -- Initializes clocks ! @@ -244,8 +252,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi Eclock_rof, EClock_wav, Eclock_esp) ! !USES: - use pio, only : file_desc_T - use shr_string_mod, only : shr_string_toupper + use pio, only : file_desc_T use shr_file_mod, only : shr_file_getunit, shr_file_freeunit use shr_mpi_mod, only : shr_mpi_bcast use seq_io_read_mod @@ -269,7 +276,6 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi type(ESMF_clock),target, intent(IN) :: EClock_wav ! wav clock type(ESMF_clock),target, intent(IN) :: EClock_esp ! esp clock type(file_desc_t) :: pioid -!EOP !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_clockInit) ' @@ -280,10 +286,11 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi type(ESMF_Time) :: StopTime1 ! Stop time type(ESMF_Time) :: StopTime2 ! Stop time type(ESMF_TimeInterval) :: TimeStep ! Clock time-step - type(ESMF_TimeInterval) :: AlarmInterval ! Alarm interval type(ESMF_CalKind_Flag) :: esmf_caltype ! local esmf calendar integer :: rc ! Return code - integer :: n ! index + integer :: n, i ! index + logical :: found + integer :: min_dt ! smallest time step integer :: dtime(max_clocks) ! time-step to use integer :: offset(max_clocks) ! run offset integer :: unitn ! i/o unit number @@ -323,6 +330,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi integer(SHR_KIND_IN) :: ice_cpl_dt ! Sea-Ice coupling interval integer(SHR_KIND_IN) :: ocn_cpl_dt ! Ocean coupling interval integer(SHR_KIND_IN) :: glc_cpl_dt ! Glc coupling interval + character(SHR_KIND_CS) :: glc_avg_period ! Glc avering coupling period integer(SHR_KIND_IN) :: rof_cpl_dt ! Runoff coupling interval integer(SHR_KIND_IN) :: wav_cpl_dt ! Wav coupling interval integer(SHR_KIND_IN) :: esp_cpl_dt ! Esp coupling interval @@ -334,8 +342,8 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi integer(SHR_KIND_IN) :: wav_cpl_offset ! Wav coupling interval integer(SHR_KIND_IN) :: rof_cpl_offset ! Runoff coupling interval integer(SHR_KIND_IN) :: esp_cpl_offset ! Esp coupling interval + logical :: esp_run_on_pause ! Run ESP on pause cycle logical :: end_restart ! Write restart at end of run - integer(SHR_KIND_IN) :: nlUnit ! Namelist unit number integer(SHR_KIND_IN) :: ierr ! Return code character(len=*), parameter :: F0A = "(2A,A)" @@ -353,22 +361,22 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi start_ymd, start_tod, ref_ymd, ref_tod, & atm_cpl_dt, ocn_cpl_dt, ice_cpl_dt, lnd_cpl_dt, & atm_cpl_offset, lnd_cpl_offset, ocn_cpl_offset, & - ice_cpl_offset, glc_cpl_dt, glc_cpl_offset, & + ice_cpl_offset, glc_cpl_dt, glc_cpl_offset, glc_avg_period, & wav_cpl_dt, wav_cpl_offset, esp_cpl_dt, esp_cpl_offset, & - rof_cpl_dt, rof_cpl_offset, end_restart + rof_cpl_dt, rof_cpl_offset, esp_run_on_pause, end_restart !------------------------------------------------------------------------------- ! Notes: !------------------------------------------------------------------------------- - SyncClock%ECP(seq_timemgr_nclock_drv)%EClock => EClock_drv - SyncClock%ECP(seq_timemgr_nclock_atm)%EClock => EClock_atm - SyncClock%ECP(seq_timemgr_nclock_lnd)%EClock => EClock_lnd - SyncClock%ECP(seq_timemgr_nclock_ocn)%EClock => EClock_ocn - SyncClock%ECP(seq_timemgr_nclock_ice)%EClock => EClock_ice - SyncClock%ECP(seq_timemgr_nclock_glc)%EClock => EClock_glc - SyncClock%ECP(seq_timemgr_nclock_rof)%EClock => EClock_rof - SyncClock%ECP(seq_timemgr_nclock_wav)%EClock => EClock_wav - SyncClock%ECP(seq_timemgr_nclock_esp)%EClock => EClock_esp + SyncClock%ECP(seq_timemgr_nclock_drv)%EClock => EClock_drv + SyncClock%ECP(seq_timemgr_nclock_atm)%EClock => EClock_atm + SyncClock%ECP(seq_timemgr_nclock_lnd)%EClock => EClock_lnd + SyncClock%ECP(seq_timemgr_nclock_ocn)%EClock => EClock_ocn + SyncClock%ECP(seq_timemgr_nclock_ice)%EClock => EClock_ice + SyncClock%ECP(seq_timemgr_nclock_glc)%EClock => EClock_glc + SyncClock%ECP(seq_timemgr_nclock_rof)%EClock => EClock_rof + SyncClock%ECP(seq_timemgr_nclock_wav)%EClock => EClock_wav + SyncClock%ECP(seq_timemgr_nclock_esp)%EClock => EClock_esp call mpi_comm_rank(mpicom,iam,ierr) @@ -415,6 +423,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ice_cpl_dt = 0 ocn_cpl_dt = 0 glc_cpl_dt = 0 + glc_avg_period = seq_timemgr_optGLCCouplingPeriod rof_cpl_dt = 0 wav_cpl_dt = 0 esp_cpl_dt = 0 @@ -426,6 +435,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi rof_cpl_offset = 0 wav_cpl_offset = 0 esp_cpl_offset = 0 + esp_run_on_pause = .true. end_restart = .true. !--------------------------------------------------------------------------- @@ -534,6 +544,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi write(logunit,F0A) trim(subname),' pause_option = ',trim(pause_option) write(logunit,F0I) trim(subname),' pause_n = ',pause_n write(logunit,F0A) trim(subname),' pause_component_list = ',trim(pause_component_list) + write(logunit,F0L) trim(subname),' esp_run_on_pause = ',esp_run_on_pause write(logunit,F0A) trim(subname),' history_option = ',trim(history_option) write(logunit,F0I) trim(subname),' history_n = ',history_n write(logunit,F0I) trim(subname),' history_ymd = ',history_ymd @@ -555,6 +566,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi write(logunit,F0I) trim(subname),' ice_cpl_dt = ',ice_cpl_dt write(logunit,F0I) trim(subname),' ocn_cpl_dt = ',ocn_cpl_dt write(logunit,F0I) trim(subname),' glc_cpl_dt = ',glc_cpl_dt + write(logunit,F0A) trim(subname),' glc_avg_period = ',glc_avg_period write(logunit,F0I) trim(subname),' rof_cpl_dt = ',rof_cpl_dt write(logunit,F0I) trim(subname),' wav_cpl_dt = ',wav_cpl_dt write(logunit,F0I) trim(subname),' esp_cpl_dt = ',esp_cpl_dt @@ -644,6 +656,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call shr_mpi_bcast( ice_cpl_dt, mpicom ) call shr_mpi_bcast( ocn_cpl_dt, mpicom ) call shr_mpi_bcast( glc_cpl_dt, mpicom ) + call shr_mpi_bcast( glc_avg_period, mpicom ) call shr_mpi_bcast( rof_cpl_dt, mpicom ) call shr_mpi_bcast( wav_cpl_dt, mpicom ) call shr_mpi_bcast( esp_cpl_dt, mpicom ) @@ -655,6 +668,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call shr_mpi_bcast( rof_cpl_offset, mpicom ) call shr_mpi_bcast( wav_cpl_offset, mpicom ) call shr_mpi_bcast( esp_cpl_offset, mpicom ) + call shr_mpi_bcast( esp_run_on_pause, mpicom ) call shr_mpi_bcast( end_restart, mpicom ) ! --- derive a couple things --- @@ -682,8 +696,60 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ! --- Initialize generic stuff --- seq_timemgr_calendar = shr_cal_calendarName(calendar) + seq_timemgr_esp_run_on_pause = esp_run_on_pause seq_timemgr_end_restart = end_restart - seq_timemgr_pause_component_list = pause_component_list + ! --- Figure out which components (if any) are doing pause this run + rc = 1 + i = 1 + if (trim(pause_component_list) == 'all') then + pause_active = .true. + else if (trim(pause_component_list) == 'none') then + pause_active = .false. + else + do + i = scan(trim(pause_component_list(rc:)), ':') - 1 + if ((i < 0) .and. (len_trim(pause_component_list) >= rc)) then + i = len_trim(pause_component_list(rc:)) + end if + if (i > 0) then + found = .false. + do n = 1, max_clocks + if (pause_component_list(rc:rc+i-1) == trim(seq_timemgr_clocks(n))) then + pause_active(n) = .true. + found = .true. + exit + end if + end do + ! Special case for cpl -- synonym for drv + if ((.not. found) .and. (pause_component_list(rc:rc+i-1) == 'cpl')) then + pause_active(seq_timemgr_nclock_drv) = .true. + found = .true. + end if + if (.not. found) then + call shr_sys_abort(subname//': unknown pause component, '//pause_component_list(rc:rc+i-1)) + end if + rc = rc + i + if (pause_component_list(rc:rc) == ':') then + rc = rc + 1 + end if + if (rc >= len_trim(pause_component_list)) then + exit + end if + else + exit + end if + end do + end if + if ( ANY(pause_active) .and. & + (trim(pause_option) /= seq_timemgr_optNONE) .and. & + (trim(pause_option) /= seq_timemgr_optNever)) then + do n = 1, max_clocks + if (pause_active(n)) then + write(logunit, '(4a)') subname, ': Pause active for ', & + trim(seq_timemgr_clocks(n)),' component' + end if + end do + end if ! --- Create the new calendar if not already set ------ if ( trim(seq_timemgr_calendar) == trim(seq_timemgr_noleap)) then @@ -706,30 +772,47 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call seq_timemgr_ETimeInit( RefTime , ref_ymd , ref_tod , "Reference date" ) call seq_timemgr_ETimeInit( CurrTime , curr_ymd , curr_tod , "Current date") - ! --- Figure out what CCSM time-stepping interval should be. --------------- + ! --- Figure out what time-stepping interval should be. --------------- dtime = 0 - dtime(seq_timemgr_nclock_atm ) = atm_cpl_dt - dtime(seq_timemgr_nclock_lnd ) = lnd_cpl_dt - dtime(seq_timemgr_nclock_ocn ) = ocn_cpl_dt - dtime(seq_timemgr_nclock_ice ) = ice_cpl_dt - dtime(seq_timemgr_nclock_glc ) = glc_cpl_dt - dtime(seq_timemgr_nclock_rof ) = rof_cpl_dt - dtime(seq_timemgr_nclock_wav ) = wav_cpl_dt - dtime(seq_timemgr_nclock_esp ) = esp_cpl_dt + dtime(seq_timemgr_nclock_atm ) = atm_cpl_dt + dtime(seq_timemgr_nclock_lnd ) = lnd_cpl_dt + dtime(seq_timemgr_nclock_ocn ) = ocn_cpl_dt + dtime(seq_timemgr_nclock_ice ) = ice_cpl_dt + dtime(seq_timemgr_nclock_glc ) = glc_cpl_dt + dtime(seq_timemgr_nclock_rof ) = rof_cpl_dt + dtime(seq_timemgr_nclock_wav ) = wav_cpl_dt + dtime(seq_timemgr_nclock_esp ) = esp_cpl_dt ! --- this finds the min of dtime excluding the driver value --- dtime(seq_timemgr_nclock_drv) = maxval(dtime) dtime(seq_timemgr_nclock_drv) = minval(dtime) + ! --- For figuring pause cycle + min_dt = maxval(dtime) + seq_timemgr_pause_sig_index = -1 + do n = 1,max_clocks if ( mod(dtime(n),dtime(seq_timemgr_nclock_drv)) /= 0) then write(logunit,*) trim(subname),' ERROR: dtime inconsistent = ',dtime call shr_sys_abort( subname//' :coupling intervals not compatible' ) endif + if (pause_active(n) .and. (dtime(n) < min_dt)) then + min_dt = dtime(n) + seq_timemgr_pause_sig_index = n + end if enddo + if (ANY(pause_active)) then + if (seq_timemgr_pause_sig_index < 1) then + write(logunit, *) subname,"ERROR: No pause_sig_index even with active pause" + call shr_sys_abort(subname//"ERROR: No pause_sig_index even with active pause") + end if + else + ! Don't try to run ESP on non-existent pauses + seq_timemgr_esp_run_on_pause = .false. + end if - ! --- Initialize clocks and alarms --- + ! --- Initialize component and driver clocks and alarms common to components and driver clocks --- do n = 1,max_clocks call ESMF_TimeIntervalSet( TimeStep, s=dtime(n), rc=rc ) @@ -739,8 +822,9 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_run), & - option = seq_timemgr_optNSeconds, & - opt_n = dtime(n), RefTime = CurrTime, & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(n), & + RefTime = CurrTime, & alarmname = trim(seq_timemgr_alarm_run)) call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & @@ -757,7 +841,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi option = seq_timemgr_optDate, & opt_ymd = stop_ymd, & opt_tod = stop_tod, & - RefTime = StartTime, & + RefTime = StartTime, & alarmname = trim(seq_timemgr_alarm_datestop)) call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & @@ -809,12 +893,21 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi endif ! Set the pause option if pause/resume is active - call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & - EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & - option = pause_option, & - opt_n = pause_n, & - RefTime = StartTime, & - alarmname = trim(seq_timemgr_alarm_pause)) + if (pause_active(n)) then + call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & + EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & + option = pause_option, & + opt_n = pause_n, & + RefTime = CurrTime, & + alarmname = trim(seq_timemgr_alarm_pause)) + else + call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & + EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & + option = seq_timemgr_optNever, & + opt_n = -1, & + RefTime = StartTime, & + alarmname = trim(seq_timemgr_alarm_pause)) + endif enddo @@ -829,15 +922,15 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ! via namelist. tcraig, 10/2007 ! -------------------------------------------------------------------- - offset(seq_timemgr_nclock_drv) = 0 - offset(seq_timemgr_nclock_atm) = atm_cpl_offset - offset(seq_timemgr_nclock_lnd) = lnd_cpl_offset - offset(seq_timemgr_nclock_ocn) = ocn_cpl_offset - offset(seq_timemgr_nclock_ice) = ice_cpl_offset - offset(seq_timemgr_nclock_glc) = glc_cpl_offset - offset(seq_timemgr_nclock_rof) = rof_cpl_offset - offset(seq_timemgr_nclock_wav) = wav_cpl_offset - offset(seq_timemgr_nclock_esp) = esp_cpl_offset + offset(seq_timemgr_nclock_drv) = 0 + offset(seq_timemgr_nclock_atm) = atm_cpl_offset + offset(seq_timemgr_nclock_lnd) = lnd_cpl_offset + offset(seq_timemgr_nclock_ocn) = ocn_cpl_offset + offset(seq_timemgr_nclock_ice) = ice_cpl_offset + offset(seq_timemgr_nclock_glc) = glc_cpl_offset + offset(seq_timemgr_nclock_rof) = rof_cpl_offset + offset(seq_timemgr_nclock_wav) = wav_cpl_offset + offset(seq_timemgr_nclock_esp) = esp_cpl_offset do n = 1,max_clocks if (abs(offset(n)) > dtime(n)) then @@ -901,16 +994,6 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_wavrun)) - call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_esp), rc=rc ) - OffsetTime = CurrTime + TimeStep - call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & - EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_esprun), & - option = seq_timemgr_optNSeconds, & - opt_n = dtime(seq_timemgr_nclock_esp), & - RefTime = OffsetTime, & - alarmname = trim(seq_timemgr_alarm_esprun)) - - ! --- this is the glcrun alarm (there ^) offset by a -dtime of the driver call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_glc), rc=rc ) OffsetTime = CurrTime + TimeStep call ESMF_TimeIntervalSet( TimeStep, s=-offset(seq_timemgr_nclock_drv), rc=rc ) @@ -921,6 +1004,23 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi opt_n = dtime(seq_timemgr_nclock_glc), & RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_glcrun)) + if (glc_avg_period == seq_timemgr_optGLCCouplingPeriod) then + ! Create this alarm identically to the glcrun alarm (which is created above) + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_glcrun_avg), & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(seq_timemgr_nclock_glc), & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_glcrun_avg)) + else if (glc_avg_period == seq_timemgr_optYearly) then + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_glcrun_avg), & + option = seq_timemgr_optYearly, & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_glcrun_avg)) + else + call shr_sys_abort(subname//':: glc_avg_period can only be glc_coupling_period or yearly') + end if call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_ocn), rc=rc ) OffsetTime = CurrTime + TimeStep @@ -943,6 +1043,15 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_ocnnext)) + call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_esp), rc=rc ) + OffsetTime = CurrTime + TimeStep + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_esprun), & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(seq_timemgr_nclock_esp), & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_esprun)) + end subroutine seq_timemgr_clockInit !=============================================================================== @@ -991,8 +1100,6 @@ subroutine seq_timemgr_EClockGetData( EClock, curr_yr, curr_mon, curr_day, & ! and reference date character(len=*) , intent(OUT), optional :: calendar ! calendar type -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_EClockGetData) ' type(ESMF_Time) :: CurrentTime ! Current time @@ -1008,9 +1115,6 @@ subroutine seq_timemgr_EClockGetData( EClock, curr_yr, curr_mon, curr_day, & integer(SHR_KIND_IN) :: ymd ! Date (YYYYMMDD) integer(SHR_KIND_IN) :: tod ! time of day (sec) integer(SHR_KIND_IN) :: ldtime ! local dtime - integer(SHR_KIND_IN) :: intyrs ! alarm variable - integer(SHR_KIND_IN) :: intmon ! alarm variable - integer(SHR_KIND_IN) :: intsec ! alarm variable integer(SHR_KIND_IN) :: days ! number of whole days in time interval integer(SHR_KIND_IN) :: seconds ! number of seconds in time interval integer(SHR_KIND_IN) :: acount ! number of valid alarms @@ -1124,8 +1228,6 @@ subroutine seq_timemgr_clockAdvance( SyncClock, force_stop, force_stop_ymd, forc integer, optional, intent(in) :: force_stop_ymd ! force stop ymd integer, optional, intent(in) :: force_stop_tod ! force stop tod -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_clockAdvance) ' integer :: n @@ -1205,6 +1307,16 @@ subroutine seq_timemgr_clockAdvance( SyncClock, force_stop, force_stop_ymd, forc call seq_timemgr_ESMFCodeCheck(rc, msg=subname//"Error from esp ESMF_ClockAdvance") endif + ! Special handling of ESP component if linked to pause cycles + if (seq_timemgr_esp_run_on_pause) then + ! We need to figure out if any pause clock is ringing + call seq_timemgr_alarmSetOff(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock,seq_timemgr_alarm_esprun) + if (seq_timemgr_alarmIsOn(SyncClock%ECP(seq_timemgr_pause_sig_index)%EClock,seq_timemgr_alarm_pause)) then + call seq_timemgr_alarmSetOn(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock,seq_timemgr_alarm_esprun) + end if + end if + + ! Special handling of restart alarm if end_restart is .true. if (seq_timemgr_end_restart) then do n = 1,max_clocks if (seq_timemgr_alarmIsOn(SyncClock%ECP(n)%EClock,seq_timemgr_alarm_stop) .or. & @@ -1241,8 +1353,6 @@ subroutine seq_timemgr_alarmInit( EClock, EAlarm, option, opt_n, opt_ymd, opt_to type(ESMF_Time) ,optional, intent(IN) :: RefTime ! ref time character(len=*) ,optional, intent(IN) :: alarmname ! alarm name -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmInit): ' integer :: rc ! Return code @@ -1254,7 +1364,6 @@ subroutine seq_timemgr_alarmInit( EClock, EAlarm, option, opt_n, opt_ymd, opt_to logical :: update_nextalarm ! update next alarm type(ESMF_Time) :: CurrTime ! Current Time type(ESMF_Time) :: NextAlarm ! Next restart alarm time - type(ESMF_Time) :: AltAlarm ! Alternate alarm time type(ESMF_TimeInterval) :: AlarmInterval ! Alarm interval !------------------------------------------------------------------------------- @@ -1485,8 +1594,6 @@ subroutine seq_timemgr_alarmGet( EAlarm, next_ymd, next_tod, prev_ymd, prev_tod, integer(SHR_KIND_IN), intent(OUT), optional :: IntYrs ! alarm int yrs character(len=*) , intent(OUT), optional :: name ! alarm name -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmGet) ' integer :: yy, mm, dd, sec ! Return time values @@ -1552,8 +1659,6 @@ subroutine seq_timemgr_AlarmSetOn( EClock, alarmname) type(ESMF_Clock), intent(INOUT) :: EClock ! clock/alarm character(len=*), intent(IN), optional :: alarmname ! alarmname -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmSetOn) ' character(len=*), parameter :: xalarm = 'unset' @@ -1562,7 +1667,6 @@ subroutine seq_timemgr_AlarmSetOn( EClock, alarmname) logical :: found logical :: set character(len=64) :: name - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1636,8 +1740,6 @@ subroutine seq_timemgr_AlarmSetOff( EClock, alarmname) type(ESMF_Clock), intent(INOUT) :: EClock ! clock/alarm character(len=*), intent(IN), optional :: alarmname ! alarmname -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmSetOff) ' character(len=*), parameter :: xalarm = 'unset' @@ -1646,7 +1748,6 @@ subroutine seq_timemgr_AlarmSetOff( EClock, alarmname) logical :: found logical :: set character(len=64) :: name - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1720,18 +1821,14 @@ logical function seq_timemgr_alarmIsOn( EClock, alarmname) type(ESMF_Clock), intent(IN) :: EClock ! clock/alarm character(len=*), intent(IN) :: alarmname ! which alarm -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmIsOn) ' character(len=*), parameter :: xalarm = 'unset' integer :: n integer :: rc logical :: found - logical :: set character(len=64) :: name type(ESMF_Time) :: ETime1, ETime2 - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1803,10 +1900,7 @@ logical function seq_timemgr_restartAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- - integer :: rc character(len=*), parameter :: subname = '(seq_timemgr_restartAlarmIsOn) ' !------------------------------------------------------------------------------- @@ -1836,8 +1930,6 @@ logical function seq_timemgr_stopAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_stopAlarmIsOn) ' @@ -1868,10 +1960,7 @@ logical function seq_timemgr_historyAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- - integer :: rc character(len=*), parameter :: subname = '(seq_timemgr_historyAlarmIsOn) ' !------------------------------------------------------------------------------- @@ -1883,6 +1972,137 @@ logical function seq_timemgr_historyAlarmIsOn( EClock) end function seq_timemgr_historyAlarmIsOn +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pauseAlarmIsOn -- check if an alarm is ringing +! +! !DESCRIPTION: +! +! check if an alarm is ringing +! +! !INTERFACE: ------------------------------------------------------------------ + +logical function seq_timemgr_pauseAlarmIsOn( EClock) + + implicit none + +! !INPUT/OUTPUT PARAMETERS: + + type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm + +!EOP + + !----- local ----- + character(len=*), parameter :: subname = '(seq_timemgr_pauseAlarmIsOn) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + seq_timemgr_pauseAlarmIsOn = & + seq_timemgr_alarmIsOn(EClock, alarmname=seq_timemgr_alarm_pause) + + end function seq_timemgr_pauseAlarmIsOn + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_active -- Is pause/resume active this run? +! +! !DESCRIPTION: +! +! Return .true. if any component is configured for pause/resume +! +! !INTERFACE: ------------------------------------------------------------------ + + logical function seq_timemgr_pause_active() + +! !INPUT/OUTPUT PARAMETERS: + +!EOP + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + seq_timemgr_pause_active = ANY(pause_active) + + end function seq_timemgr_pause_active + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_component_index -- return an index for a component +! +! !DESCRIPTION: +! +! Look up a component's internal index for faster processing +! +! !INTERFACE: ------------------------------------------------------------------ + + integer function seq_timemgr_pause_component_index(component_name) + +! !INPUT/OUTPUT PARAMETERS: + + character(len=*), intent(IN) :: component_name + +!EOP + + !----- local ----- + integer :: ind + character(len=*), parameter :: subname = '(seq_timemgr_pause_component_index) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + seq_timemgr_pause_component_index = 0 + do ind = 1, max_clocks + if (trim(component_name) == trim(seq_timemgr_clocks(ind))) then + seq_timemgr_pause_component_index = ind + exit + end if + end do + if (seq_timemgr_pause_component_index < 1) then + if (trim(component_name) == 'cpl') then + seq_timemgr_pause_component_index = seq_timemgr_nclock_drv + end if + end if + if (seq_timemgr_pause_component_index < 1) then + call shr_sys_abort(subname//': No index for component '//trim(component_name)) + end if + + end function seq_timemgr_pause_component_index + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_component_active -- Check if component paused +! +! !DESCRIPTION: +! +! Return .true. if component is active in driver pause +! +! !INTERFACE: ------------------------------------------------------------------ + + logical function seq_timemgr_pause_component_active(component_index) + +! !INPUT/OUTPUT PARAMETERS: + + integer, intent(IN) :: component_index + +!EOP + + !----- local ----- + character(len=*), parameter :: subname = '(seq_timemgr_pause_component_active) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + if ((component_index < 1) .or. (component_index > max_clocks)) then + call shr_sys_abort(subname//': component_index out of range') + end if + seq_timemgr_pause_component_active = pause_active(component_index) + + end function seq_timemgr_pause_component_active + !=============================================================================== !=============================================================================== ! !IROUTINE: seq_timemgr_ETimeInit -- Create ESMF_Time object based on YMD values @@ -1906,8 +2126,6 @@ subroutine seq_timemgr_ETimeInit( ETime, ymd, tod, desc ) integer , intent(in), optional :: tod ! Time of day in seconds character(len=*), intent(in), optional :: desc ! Description of time to set -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ETimeInit) ' integer :: yr, mon, day ! Year, month, day as integers @@ -1965,8 +2183,6 @@ subroutine seq_timemgr_ETimeGet( ETime, offset, ymd, tod ) integer, optional, intent(OUT) :: ymd ! date of day integer, optional, intent(OUT) :: tod ! Time of day -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ETimeGet) ' type(ESMF_Time) :: ETimeAdd ! ESMF time + offset @@ -2035,8 +2251,6 @@ subroutine seq_timemgr_EClockInit( TimeStep, StartTime, RefTime, CurrTime, ECloc type(ESMF_Time) , intent(IN) :: CurrTime ! Current time type(ESMF_Clock) , intent(OUT) :: EClock ! Output ESMF clock -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_EClockInit) ' integer :: rc ! ESMF return code @@ -2098,8 +2312,6 @@ logical function seq_timemgr_EClockDateInSync( EClock, ymd, tod, prev) integer, intent(IN) :: tod ! Time of day (sec) logical, optional,intent(IN) :: prev ! If should get previous time -!EOP - !----- local ----- character(len=*), parameter :: subname = "(seq_timemgr_EClockDateInSync) " type(ESMF_Time) :: ETime @@ -2150,9 +2362,7 @@ subroutine seq_timemgr_clockPrint( SyncClock ) ! !INPUT/OUTPUT PARAMETERS: - type(seq_timemgr_type), intent(in) :: SyncClock ! Input clock to print - -!EOP + type(seq_timemgr_type), intent(in) :: SyncClock ! Input clock to print character(len=*), parameter :: subname = "(seq_timemgr_clockPrint) " integer(SHR_KIND_IN) :: m,n @@ -2258,8 +2468,6 @@ subroutine seq_timemgr_ESMFDebug( EClock, ETime, ETimeInterval, istring ) type(ESMF_TimeInterval), optional, intent(inout) :: ETimeInterval ! ESMF Time Interval character(len=*), optional, intent(in) :: istring -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ESMFDebug) ' character(len=128) :: timestring @@ -2332,8 +2540,6 @@ subroutine seq_timemgr_ESMFCodeCheck( rc, msg ) integer, intent(in) :: rc ! return code from ESMF character(len=*),optional,intent(in) :: msg ! error message -!EOP - character(len=*),parameter :: subname = 'seq_timemgr_ESMFCodeCheck' !------------------------------------------------------------------------------- ! Notes: diff --git a/src/drivers/mct/shr/shr_ndep_mod.F90 b/src/drivers/mct/shr/shr_ndep_mod.F90 new file mode 100644 index 000000000000..64cadf107f35 --- /dev/null +++ b/src/drivers/mct/shr/shr_ndep_mod.F90 @@ -0,0 +1,117 @@ +module shr_ndep_mod + + !======================================================================== + ! Module for handling nitrogen depostion of tracers. + ! This module is shared by land and atmosphere models for the computations of + ! dry deposition of tracers + !======================================================================== + + !USES: + use shr_sys_mod, only : shr_sys_abort + use shr_log_mod, only : s_loglev => shr_log_Level + use shr_kind_mod, only : r8 => shr_kind_r8, CS => SHR_KIND_CS, CX => SHR_KIND_CX + + implicit none + save + + private + + ! !PUBLIC MEMBER FUNCTIONS + public :: shr_ndep_readnl ! Read namelist + !==================================================================================== + +CONTAINS + + !==================================================================================== + subroutine shr_ndep_readnl(NLFilename, ID, ndep_fields, add_ndep_fields) + + !======================================================================== + ! reads ndep_inparm namelist and sets up driver list of fields for + ! atmosphere -> land and atmosphere -> ocn communications. + !======================================================================== + + use shr_file_mod , only : shr_file_getUnit, shr_file_freeUnit + use shr_log_mod , only : s_logunit => shr_log_Unit + use seq_comm_mct , only : seq_comm_iamroot, seq_comm_setptrs + use shr_mpi_mod , only : shr_mpi_bcast + use shr_nl_mod , only : shr_nl_find_group_name + implicit none + + character(len=*), intent(in) :: NLFilename ! Namelist filename + integer , intent(in) :: ID ! seq_comm ID + character(len=*), intent(out) :: ndep_fields + logical , intent(out) :: add_ndep_fields + + !----- local ----- + integer :: i ! Indices + integer :: unitn ! namelist unit number + integer :: ierr ! error code + logical :: exists ! if file exists or not + character(len=8) :: token ! dry dep field name to add + integer :: mpicom ! MPI communicator + + integer, parameter :: maxspc = 100 ! Maximum number of species + character(len=32) :: ndep_list(maxspc) = '' ! List of ndep species + + !----- formats ----- + character(*),parameter :: subName = '(shr_ndep_read) ' + character(*),parameter :: F00 = "('(shr_ndep_read) ',8a)" + character(*),parameter :: FI1 = "('(shr_ndep_init) ',a,I2)" + + namelist /ndep_inparm/ ndep_list + + !----------------------------------------------------------------------------- + ! Read namelist and figure out the ndep field list to pass + ! First check if file exists and if not, n_ndep will be zero + !----------------------------------------------------------------------------- + + !--- Open and read namelist --- + if ( len_trim(NLFilename) == 0 ) then + call shr_sys_abort( subName//'ERROR: nlfilename not set' ) + end if + call seq_comm_setptrs(ID,mpicom=mpicom) + if (seq_comm_iamroot(ID)) then + inquire( file=trim(NLFileName), exist=exists) + if ( exists ) then + unitn = shr_file_getUnit() + open( unitn, file=trim(NLFilename), status='old' ) + if ( s_loglev > 0 ) then + write(s_logunit,F00) 'Read in ndep_inparm namelist from: ', trim(NLFilename) + end if + call shr_nl_find_group_name(unitn, 'ndep_inparm', ierr) + if (ierr == 0) then + ierr = 1 + do while ( ierr /= 0 ) + read(unitn, ndep_inparm, iostat=ierr) + if (ierr < 0) then + call shr_sys_abort( subName//'ERROR: encountered end-of-file on namelist read' ) + endif + end do + close( unitn ) + else + write(s_logunit,*) 'shr_ndep_readnl: no ndep_inparm namelist found in ',NLFilename + endif + call shr_file_freeUnit( unitn ) + end if + end if + call shr_mpi_bcast( ndep_list, mpicom ) + + ndep_fields = ' ' + if (len_trim(ndep_list(1)) == 0) then + add_ndep_fields = .false. + else + ! Loop over species to fill list of fields to communicate for ndep + add_ndep_fields = .true. + do i=1,maxspc + if ( len_trim(ndep_list(i))==0 ) exit + if ( i == 1 ) then + ndep_fields = 'Faxa_' // trim(ndep_list(i)) + else + ndep_fields = trim(ndep_fields)//':'//'Faxa_' // trim(ndep_list(i)) + endif + enddo + end if + + end subroutine shr_ndep_readnl + +end module shr_ndep_mod diff --git a/src/drivers/mct/unit_test/CMakeLists.txt b/src/drivers/mct/unit_test/CMakeLists.txt index 82603c37bfd4..a747f9e23a2b 100644 --- a/src/drivers/mct/unit_test/CMakeLists.txt +++ b/src/drivers/mct/unit_test/CMakeLists.txt @@ -11,6 +11,11 @@ add_definitions( -DNUM_COMP_INST_ESP=1 ) +# The following definitions are needed when building with the mpi-serial library +if (USE_MPI_SERIAL) + add_definitions(-DNO_MPI2 -DNO_MPIMOD) +endif() + # Add source directories from stubs. This should be done first, so that in the # case of name collisions, the drv versions take precedence (when there are two # files with the same name, the one added later wins). @@ -46,6 +51,9 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) # tests need all of these libraries, but it's easiest just to set the same list # for everyone. set(DRV_UNIT_TEST_LIBS drv;csm_share;esmf_wrf_timemgr;mct;mpeu) +if (USE_MPI_SERIAL) + list(APPEND DRV_UNIT_TEST_LIBS mpi-serial) +endif() list(APPEND DRV_UNIT_TEST_LIBS ${NETCDF_LIBRARIES}) # Add the test directories diff --git a/src/drivers/mct/unit_test/stubs/CMakeLists.txt b/src/drivers/mct/unit_test/stubs/CMakeLists.txt index 4e0db12da1a9..a3097917ed99 100644 --- a/src/drivers/mct/unit_test/stubs/CMakeLists.txt +++ b/src/drivers/mct/unit_test/stubs/CMakeLists.txt @@ -1,4 +1,5 @@ list(APPEND drv_sources + seq_timemgr_mod.F90 vertical_gradient_calculator_constant.F90 ) diff --git a/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 b/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 new file mode 100644 index 000000000000..f88a96d2da8c --- /dev/null +++ b/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 @@ -0,0 +1,19 @@ +module seq_timemgr_mod + + ! Stub for routines from seq_timemgr_mod that are needed by other modules built by the + ! unit tests. + + implicit none + private + + public :: seq_timemgr_pause_active + +contains + + logical function seq_timemgr_pause_active() + ! Stub for seq_timemgr_pause_active - always returns .false. + + seq_timemgr_pause_active = .false. + end function seq_timemgr_pause_active + +end module seq_timemgr_mod diff --git a/src/externals/CMake/CIME_initial_setup.cmake b/src/externals/CMake/CIME_initial_setup.cmake new file mode 100644 index 000000000000..c3e833175749 --- /dev/null +++ b/src/externals/CMake/CIME_initial_setup.cmake @@ -0,0 +1,16 @@ +# Module used for CIME testing. +# +# This module does some initial setup that must be done BEFORE the 'project' +# line in the main CMakeLists.txt file. + +include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND) +if(NOT FOUND) + message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure") +endif() +if("$ENV{MPILIB}" STREQUAL "mpi-serial") + set(CMAKE_C_COMPILER ${SCC}) + set(CMAKE_Fortran_COMPILER ${SFC}) +else() + set(CMAKE_C_COMPILER ${MPICC}) + set(CMAKE_Fortran_COMPILER ${MPIFC}) +endif() diff --git a/src/externals/CMake/CIME_utils.cmake b/src/externals/CMake/CIME_utils.cmake index eeca4ae464fd..bd0f254b1d8f 100644 --- a/src/externals/CMake/CIME_utils.cmake +++ b/src/externals/CMake/CIME_utils.cmake @@ -3,6 +3,10 @@ # This module contains statements that would otherwise be boilerplate in # most CIME tests. It enables CTest testing, handles the USE_COLOR and # ENABLE_GENF90 arguments, and includes several other modules. +# +# Some of the things done here must be done AFTER the 'project' line in the main +# CMakeLists.txt file. This assumes that CIME_initial_setup has already been +# included. #========================================================================== # Copyright (c) 2013-2014, University Corporation for Atmospheric Research @@ -30,16 +34,10 @@ set(CMAKE_COLOR_MAKEFILE "${USE_COLOR}") # Compiler info #================================================= -include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND) list(APPEND CMAKE_MODULE_PATH "../pio2/cmake") -set(CMAKE_C_COMPILER ${MPICC}) -set(CMAKE_Fortran_COMPILER ${MPIF90}) set(CMAKE_C_FLAGS "${CPPDEFS} ${CFLAGS}") set(CMAKE_Fortran_FLAGS "${CPPDEFS} ${FFLAGS}") -set (CMAKE_EXE_LINKER_FLAGS ${LDFLAGS}) -if(NOT FOUND) - message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure") -endif() +set(CMAKE_EXE_LINKER_FLAGS "${LDFLAGS} ${SLIBS}") include(Compilers) diff --git a/src/externals/CMake/README.md b/src/externals/CMake/README.md index 21d36fb3be65..ae583206b0e9 100644 --- a/src/externals/CMake/README.md +++ b/src/externals/CMake/README.md @@ -28,6 +28,12 @@ Sourcelist_utils - Use source file lists defined over multiple directories. Modules that are CESM-specific and/or incomplete: -CIME_utils - Handles a few options, and includes several other modules. +CIME\_initial\_setup - Handles setup that must be done before the 'project' +line. This must be included before the 'project' line in the main CMakeLists.txt +file. + +CIME_utils - Handles a few options, and includes several other modules. This +must be included after the 'project' line in the main CMakeLists.txt file, and +after the inclusion of CIME\_initial\_setup. Compilers - Specify compiler-specific behavior, add build types for CESM. diff --git a/src/externals/CMake/pFUnit_utils.cmake b/src/externals/CMake/pFUnit_utils.cmake index 0089e4989f42..2accabe20e14 100644 --- a/src/externals/CMake/pFUnit_utils.cmake +++ b/src/externals/CMake/pFUnit_utils.cmake @@ -215,7 +215,8 @@ function(create_pFUnit_test test_name executable_name pf_file_list fortran_sourc endif() # Prefix command with an mpirun command - set (MY_COMMAND ${PFUNIT_MPIRUN} ${MY_COMMAND}) + separate_arguments(PFUNIT_MPIRUN_LIST UNIX_COMMAND ${PFUNIT_MPIRUN}) + set (MY_COMMAND ${PFUNIT_MPIRUN_LIST} ${MY_COMMAND}) # Do the work add_pFUnit_executable(${executable_name} "${pf_file_list}" diff --git a/src/externals/pio1/pio/CMakeLists.txt b/src/externals/pio1/pio/CMakeLists.txt index a23ee13b5487..d2c162bb71d1 100644 --- a/src/externals/pio1/pio/CMakeLists.txt +++ b/src/externals/pio1/pio/CMakeLists.txt @@ -189,5 +189,3 @@ endif() if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../testpio) ADD_SUBDIRECTORY(../testpio testpio) endif() - - diff --git a/src/externals/pio2/CMakeLists.txt b/src/externals/pio2/CMakeLists.txt index 6677ab46eb09..514384a5a9e6 100644 --- a/src/externals/pio2/CMakeLists.txt +++ b/src/externals/pio2/CMakeLists.txt @@ -46,29 +46,6 @@ else() set(USE_MPI_SERIAL 0) endif() -#===== Library Variables ===== -set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") - -#===== Testing Options ===== -option (PIO_ENABLE_TESTS "Enable the testing builds" ON) -option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) - -#============================================================================== -# BACKWARDS COMPATIBILITY -#============================================================================== - -# Old NETCDF_DIR variable --> NetCDF_PATH -if (DEFINED NETCDF_DIR) - set (NetCDF_PATH ${NETCDF_DIR} - CACHE STRING "Location of the NetCDF library installation") -endif () - -# Old PNETCDF_DIR variable --> PnetCDF_PATH -if (DEFINED PNETCDF_DIR) - set (PnetCDF_PATH ${PNETCDF_DIR} - CACHE STRING "Location of the PnetCDF library installation") -endif () - #============================================================================== # PREPEND TO CMAKE MODULE PATH #============================================================================== @@ -97,6 +74,46 @@ set (USER_CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH} CACHE STRING "Location of the CMake_Fortran_utils") list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH}) +INCLUDE (CheckTypeSize) + +#===== MPI ===== +if (PIO_USE_MPISERIAL) + find_package (MPISERIAL COMPONENTS C REQUIRED) + if (MPISERIAL_C_FOUND) + set (CMAKE_REQUIRED_INCLUDES ${MPISERIAL_C_INCLUDE_DIRS}) + endif () +else () + find_package (MPI REQUIRED) + set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) +endif () + +SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h") +check_type_size("MPI_Offset" SIZEOF_MPI_OFFSET) +SET(CMAKE_EXTRA_INCLUDE_FILES) + +#===== Library Variables ===== +set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") + +#===== Testing Options ===== +option (PIO_ENABLE_TESTS "Enable the testing builds" ON) +option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) + +#============================================================================== +# BACKWARDS COMPATIBILITY +#============================================================================== + +# Old NETCDF_DIR variable --> NetCDF_PATH +if (DEFINED NETCDF_DIR) + set (NetCDF_PATH ${NETCDF_DIR} + CACHE STRING "Location of the NetCDF library installation") +endif () + +# Old PNETCDF_DIR variable --> PnetCDF_PATH +if (DEFINED PNETCDF_DIR) + set (PnetCDF_PATH ${PNETCDF_DIR} + CACHE STRING "Location of the PnetCDF library installation") +endif () + #============================================================================== # HELPFUL GLOBAL VARIABLES #============================================================================== diff --git a/src/externals/pio2/CTestScript.cmake b/src/externals/pio2/CTestScript.cmake index 1d0de3ccdcdd..e817f422d158 100644 --- a/src/externals/pio2/CTestScript.cmake +++ b/src/externals/pio2/CTestScript.cmake @@ -18,7 +18,7 @@ else () set (CTEST_DASHBOARD_ROOT "$ENV{HOME}/pio-dashboard") endif () -## -- Compiler ID +## -- Compiler ID if (DEFINED ENV{PIO_COMPILER_ID}) set (compid "$ENV{PIO_COMPILER_ID}") else () @@ -49,7 +49,8 @@ if (HOSTNAME MATCHES "^yslogin" OR HOSTNAME MATCHES "^pronghorn") set (HOSTNAME_ID "nwsc") # New UCAR/NWSC SGI Machines -elseif (HOSTNAME MATCHES "^laramie") +elseif (HOSTNAME MATCHES "^laramie" OR + HOSTNAME MATCHES "^chadmin") set (HOSTNAME_ID "nwscla") # ALCF/Argonne Machines elseif (HOSTNAME MATCHES "^mira" OR @@ -109,9 +110,9 @@ find_program (CTEST_GIT_COMMAND NAMES git) ## -- make command find_program (MAKE NAMES make) -#----------------------------------------------------------- +#----------------------------------------------------------- #-- Generate build-specific information -#----------------------------------------------------------- +#----------------------------------------------------------- ## -- CTest Site Name @@ -124,25 +125,25 @@ set (CTEST_BUILD_NAME "${osname}-${osrel}-${cpu}-${compid}") ## -- SRC Dir (where this script exists) set (CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}") -## -- BIN Dir +## -- BIN Dir set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/build-${CTEST_BUILD_NAME}-${CTEST_BUILD_GROUP}") ## -- Add the CTest script directory to the module path set (CTEST_EXTRA_SCRIPT_PATH "${CTEST_SOURCE_DIRECTORY}/ctest") list (APPEND CMAKE_MODULE_PATH ${CTEST_EXTRA_SCRIPT_PATH}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Store Build-Specific Info (environment variables) -# ----------------------------------------------------------- +# ----------------------------------------------------------- set (ENV{PIO_DASHBOARD_SITE} ${CTEST_SITE}) set (ENV{PIO_DASHBOARD_BUILD_NAME} ${CTEST_BUILD_NAME}) set (ENV{PIO_DASHBOARD_SOURCE_DIR} ${CTEST_SOURCE_DIRECTORY}) set (ENV{PIO_DASHBOARD_BINARY_DIR} ${CTEST_BINARY_DIRECTORY}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Run CTest -# ----------------------------------------------------------- +# ----------------------------------------------------------- ## -- Empty the binary directory ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) @@ -157,7 +158,7 @@ message (" -- Update source - ${CTEST_BUILD_NAME} --") set (CTEST_UPDATE_COMMAND "${CTEST_GIT_COMMAND}") ctest_update () -## -- Configure +## -- Configure message (" -- Configure build - ${CTEST_BUILD_NAME} --") include (CTestEnvironment-${HOSTNAME_ID}) set (CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} ${CTEST_CONFIGURE_OPTIONS} ${CTEST_SOURCE_DIRECTORY}") @@ -179,9 +180,9 @@ message (" -- Submit to dashboard - ${CTEST_BUILD_NAME} --") message ("** -- PIO_DASHBOARD_SITE=$ENV{PIO_DASHBOARD_SITE}") ctest_submit () -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Clear environment -# ----------------------------------------------------------- +# ----------------------------------------------------------- unset (ENV{PIO_DASHBOARD_SITE}) unset (ENV{PIO_DASHBOARD_BUILD_NAME}) diff --git a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake index 07ba92a2a726..ddf04f063a6e 100644 --- a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake @@ -23,3 +23,8 @@ endif () if (DEFINED ENV{VALGRIND_CHECK}) set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON") endif () + +# If USE_MALLOC environment variable is set, then use native malloc (instead of bget package) +if (DEFINED ENV{USE_MALLOC}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MALLOC=ON") +endif () diff --git a/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake index 356390f933d6..4a0d6fd3acd5 100644 --- a/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake @@ -14,5 +14,5 @@ set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) - set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON") + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON -DPIO_ENABLE_EXAMPLES=OFF ") endif () diff --git a/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh new file mode 100755 index 000000000000..56407d9c4251 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module purge +module load compiler/gnu/5.4.0 +module load tool/parallel-netcdf/1.8.1/gnu-5.4.0/openmpi + +export CC=mpicc +export FC=mpif90 +export PIO_DASHBOARD_SITE="cgd" +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard +export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src +export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} +export PIO_COMPILER_ID=gcc-`gcc --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} +echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + + +ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-cgd-nag.sh b/src/externals/pio2/ctest/runcdash-cgd-nag.sh index 55c477b73bce..e413186131af 100755 --- a/src/externals/pio2/ctest/runcdash-cgd-nag.sh +++ b/src/externals/pio2/ctest/runcdash-cgd-nag.sh @@ -14,7 +14,7 @@ module load tool/parallel-netcdf/1.7.0/nag/mvapich2 export CC=mpicc export FC=mpif90 export PIO_DASHBOARD_SITE="cgd" -export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` diff --git a/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh index 702427e5a7b8..68ac5826be9f 100755 --- a/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh @@ -14,7 +14,7 @@ module load git/2.3.0 module load cmake/3.0.2 module load netcdf/4.3.3.1 -export MPISERIAL=/glade/p/work/katec/installs/intel_15.0.3 +export MPISERIAL=/glade/u/home/jedwards/mpi-serial/intel15.0.3/ export CC=icc export FC=ifort diff --git a/src/externals/pio2/ctest/runcdash-nwscla-intel.sh b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh index 3812b9d61f01..630ac8380859 100755 --- a/src/externals/pio2/ctest/runcdash-nwscla-intel.sh +++ b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh @@ -11,18 +11,18 @@ source /etc/profile.d/modules.sh module reset module unload netcdf -module swap intel intel/16.0.3 -module load git/2.10.0 -module load cmake/3.6.2 -module load netcdf-mpi/4.4.1 +module swap intel intel/17.0.1 +module load cmake/3.7.2 +module load netcdf-mpi/4.4.1.1 module load pnetcdf/1.8.0 +module switch mpt mpt/2.15 echo "MODULE LIST..." module list export CC=mpicc export FC=mpif90 -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_DASHBOARD_ROOT=/glade/scratch/jedwards/dashboard export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then diff --git a/src/externals/pio2/ctest/runctest-cgd.sh b/src/externals/pio2/ctest/runctest-cgd.sh index fccf3396d3b1..bbd31ccf5d0f 100755 --- a/src/externals/pio2/ctest/runctest-cgd.sh +++ b/src/externals/pio2/ctest/runctest-cgd.sh @@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh -q short` +jobid=`/usr/local/bin/qsub -l nodes=1:ppn=8 runctest.sh -q short` # Wait for the job to complete before exiting while true; do diff --git a/src/externals/pio2/ctest/runctest-nwscla.sh b/src/externals/pio2/ctest/runctest-nwscla.sh index 6b6f4d87f7c8..d3e252317d05 100755 --- a/src/externals/pio2/ctest/runctest-nwscla.sh +++ b/src/externals/pio2/ctest/runctest-nwscla.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the NCAR Wyoming Supercomputing -# Center systems (yellowstone/caldera/geyser). +# This script defines how to run CTest on the NCAR Wyoming Supercomputing +# Center systems (cheyenne/laramie). # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -19,7 +19,8 @@ model=$2 echo "#!/bin/sh" > runctest.sh echo "#PBS -l walltime=01:00:00" >> runctest.sh echo "#PBS -l select=1:ncpus=8:mpiprocs=8" >> runctest.sh -echo "#PBS -A SCSG0002" >> runctest.sh +echo "#PBS -A P93300606" >> runctest.sh +echo "#PBS -q regular" >> runctest.sh echo "export PIO_DASHBOARD_SITE=nwscla-${HOSTNAME}" >> runctest.sh echo "CTESTCMD=`which ctest`" >> runctest.sh echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh @@ -28,18 +29,16 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`qsub runctest.sh` +jobid=`qsub -l walltime=01:00:00 runctest.sh` # Wait for the job to complete before exiting while true; do - status=`qstat $jobid` - echo $status - if [ "$status" == "" ]; then - break + qstat $jobid + if [ $? -eq 0 ]; then + sleep 30 else - sleep 10 + break; fi done exit 0 - diff --git a/src/externals/pio2/doc/source/contributing_code.txt b/src/externals/pio2/doc/source/contributing_code.txt index 6dbd70898264..c811777a02b8 100644 --- a/src/externals/pio2/doc/source/contributing_code.txt +++ b/src/externals/pio2/doc/source/contributing_code.txt @@ -13,7 +13,7 @@ * Documents produced by Doxygen are derivative works derived from the * input used in their production; they are not affected by this license. * - */ /*! \page code_style Code Style for Contributors + */ /*! \page contributing_code Guide for Contributors # Introduction # diff --git a/src/externals/pio2/examples/c/CMakeLists.txt b/src/externals/pio2/examples/c/CMakeLists.txt index 74b4837c7f1d..e35562b11cc1 100644 --- a/src/externals/pio2/examples/c/CMakeLists.txt +++ b/src/externals/pio2/examples/c/CMakeLists.txt @@ -33,12 +33,22 @@ ADD_EXECUTABLE(example1 example1.c) TARGET_LINK_LIBRARIES(example1 pioc) add_dependencies(tests example1) +ADD_EXECUTABLE(darray_no_async darray_no_async.c) +TARGET_LINK_LIBRARIES(darray_no_async pioc) +add_dependencies(tests darray_no_async) + +ADD_EXECUTABLE(darray_async darray_async.c) +TARGET_LINK_LIBRARIES(darray_async pioc) +add_dependencies(tests darray_async) + if (PIO_USE_MPISERIAL) add_test(NAME examplePio COMMAND examplePio) add_test(NAME example1 COMMAND example1) else () add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60) add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60) + #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60) + add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60) endif () diff --git a/src/externals/pio2/examples/c/darray_async.c b/src/externals/pio2/examples/c/darray_async.c new file mode 100644 index 000000000000..1b15607f0f62 --- /dev/null +++ b/src/externals/pio2/examples/c/darray_async.c @@ -0,0 +1,387 @@ +/* + * @file + * @brief A simple C example for the ParallelIO Library. + * + * This example creates a netCDF output file with three dimensions + * (one unlimited) and one variable. It first writes, then reads the + * sample file using distributed arrays. + * + * This example can be run in parallel for 4 processors. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef TIMING +#include +#endif + +/* The name of this program. */ +#define TEST_NAME "darray_async" + +/* The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/* The number of dimensions in the example data. */ +#define NDIM3 3 + +/* The number of timesteps of data. */ +#define NUM_TIMESTEPS 2 + +/* The length of our sample data in X dimension.*/ +#define DIM_LEN_X 4 + +/* The length of our sample data in Y dimension.*/ +#define DIM_LEN_Y 4 + +/* The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/* Return code when netCDF output file does not match + * expectations. */ +#define ERR_BAD 1001 + +/* The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/* Number of tasks this example runs on. */ +#define TARGET_NTASKS 5 + +/* Logging level. */ +#define LOG_LEVEL 3 + +/* Number of IO processors. */ +#define NUM_IO_TASKS 1 + +/* Number of computation processors. */ +#define NUM_COMP_TASKS 4 + +/* Number of computation components. */ +#define COMPONENT_COUNT 1 + +/* Lengths of dimensions. */ +int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + +/* Names of dimensions. */ +char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; + +/* Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/* Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/* Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/* This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/* @brief Check the output file. + * + * Use netCDF to check that the output is as expected. + * + * @param ntasks The number of processors running the example. + * @param filename The name of the example file to check. + * + * @return 0 if example file is correct, non-zero otherwise. */ +/* int check_file(int iosysid, int ntasks, char *filename, int iotype, */ +/* int elements_per_pe, int my_rank, int ioid) */ +/* { */ + +/* int ncid; /\* File ID from netCDF. *\/ */ +/* int ndims; /\* Number of dimensions. *\/ */ +/* int nvars; /\* Number of variables. *\/ */ +/* int ngatts; /\* Number of global attributes. *\/ */ +/* int unlimdimid; /\* ID of unlimited dimension. *\/ */ +/* int natts; /\* Number of variable attributes. *\/ */ +/* nc_type xtype; /\* NetCDF data type of this variable. *\/ */ +/* int ret; /\* Return code for function calls. *\/ */ +/* int dimids[NDIM3]; /\* Dimension ids for this variable. *\/ */ +/* char var_name[NC_MAX_NAME]; /\* Name of the variable. *\/ */ +/* /\* size_t start[NDIM3]; /\\* Zero-based index to start read. *\\/ *\/ */ +/* /\* size_t count[NDIM3]; /\\* Number of elements to read. *\\/ *\/ */ +/* /\* int buffer[DIM_LEN_X]; /\\* Buffer to read in data. *\\/ *\/ */ +/* /\* int expected[DIM_LEN_X]; /\\* Data values we expect to find. *\\/ *\/ */ + +/* /\* Open the file. *\/ */ +/* if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) */ +/* return ret; */ +/* printf("opened file %s ncid = %d\n", filename, ncid); */ + +/* /\* Check the metadata. *\/ */ +/* if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) */ +/* return ret; */ + +/* /\* Check the dimensions. *\/ */ +/* if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) */ +/* return ERR_BAD; */ +/* for (int d = 0; d < NDIM3; d++) */ +/* { */ +/* char my_dim_name[NC_MAX_NAME]; */ +/* PIO_Offset dimlen; */ + +/* if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */ +/* return ret; */ +/* if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) */ +/* return ERR_BAD; */ +/* } */ + +/* /\* Check the variable. *\/ */ +/* if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) */ +/* return ret; */ +/* if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || */ +/* dimids[2] != 2 || natts != 0) */ +/* return ERR_BAD; */ + +/* /\* Allocate storage for sample data. *\/ */ +/* int buffer[elements_per_pe]; */ +/* int buffer_in[elements_per_pe]; */ + +/* /\* Check each timestep. *\/ */ +/* for (int t = 0; t < NUM_TIMESTEPS; t++) */ +/* { */ +/* int varid = 0; /\* There's only one var in sample file. *\/ */ + +/* /\* This is the data we expect for this timestep. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ + +/* /\* Read one record. *\/ */ +/* if ((ret = PIOc_setframe(ncid, varid, t))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) */ +/* return ret; */ + +/* /\* Check the results. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* if (buffer_in[i] != buffer[i]) */ +/* return ERR_BAD; */ +/* } */ + +/* /\* Close the file. *\/ */ +/* if ((ret = PIOc_closefile(ncid))) */ +/* return ret; */ + +/* /\* Everything looks good! *\/ */ +/* return 0; */ +/* } */ + +/* Write, then read, a simple example with darrays. + + The sample file created by this program is a small netCDF file. It + has the following contents (as shown by ncdump): + + 
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+
+ +*/ + int main(int argc, char* argv[]) + { + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int iosysid; /* The ID for the parallel I/O system. */ + /* int ncid; /\* The ncid of the netCDF file. *\/ */ + /* int dimid[NDIM3]; /\* The dimension ID. *\/ */ + /* int varid; /\* The ID of the netCDF varable. *\/ */ + /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */ + /* int num_flavors = 0; /\* Number of iotypes available in this build. *\/ */ + /* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */ + int ret; /* Return value. */ + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + printf("%d: ParallelIO Library darray_async example running on %d processors.\n", + my_rank, ntasks); + if (ntasks != TARGET_NTASKS) + { + fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS); + return ERR_BAD; + } + + /* Turn on logging. */ + if ((ret = PIOc_set_log_level(LOG_LEVEL))) + return ret; + + /* Num procs for computation. */ + int num_procs2[COMPONENT_COUNT] = {4}; + + /* Is the current process a computation task? */ + int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1; + + /* Initialize the IO system. */ + if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT, + num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))) + ERR(ret); + + + /* The rest of the code executes on computation tasks only. As + * PIO functions are called on the computation tasks, the + * async system will call them on the IO task. When the + * computation tasks call PIO_finalize(), the IO task will get + * a message to shut itself down. */ + if (comp_task) + { + /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */ + /* int ioid; /\* The I/O description ID. *\/ */ + + /* /\* How many elements on each computation task? *\/ */ + /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */ + + /* /\* Allocate and initialize array of decomposition mapping. *\/ */ + /* PIO_Offset compdof[elements_per_pe]; */ + /* for (int i = 0; i < elements_per_pe; i++) */ + /* compdof[i] = my_rank * elements_per_pe + i; */ + + /* /\* Create the PIO decomposition for this example. Since */ + /* this is a variable with an unlimited dimension, we want */ + /* to create a 2-D composition which represents one */ + /* record. *\/ */ + /* printf("rank: %d Creating decomposition...\n", my_rank); */ + /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */ + /* compdof, &ioid, 0, NULL, NULL))) */ + /* ERR(ret); */ + +/* /\* The number of favors may change with the build parameters. *\/ */ +/* #ifdef _PNETCDF */ +/* format[num_flavors++] = PIO_IOTYPE_PNETCDF; */ +/* #endif */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF; */ +/* #ifdef _NETCDF4 */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF4C; */ +/* format[num_flavors++] = PIO_IOTYPE_NETCDF4P; */ +/* #endif */ + +/* /\* Use PIO to create the example file in each of the four */ +/* * available ways. *\/ */ +/* for (int fmt = 0; fmt < num_flavors; fmt++) */ +/* { */ +/* /\* Create a filename. *\/ */ +/* sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); */ + +/* /\* Create the netCDF output file. *\/ */ +/* printf("rank: %d Creating sample file %s with format %d...\n", */ +/* my_rank, filename, format[fmt]); */ +/* if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) */ +/* ERR(ret); */ + +/* /\* Define netCDF dimension and variable. *\/ */ +/* printf("rank: %d Defining netCDF metadata...\n", my_rank); */ +/* for (int d = 0; d < NDIM3; d++) */ +/* if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_enddef(ncid))) */ +/* ERR(ret); */ + +/* /\* Allocate storage for sample data. *\/ */ +/* int buffer[elements_per_pe]; */ + +/* /\* Write each timestep. *\/ */ +/* for (int t = 0; t < NUM_TIMESTEPS; t++) */ +/* { */ +/* /\* Create some data for this timestep. *\/ */ +/* for (int i = 0; i < elements_per_pe; i++) */ +/* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ + +/* /\* Write data to the file. *\/ */ +/* printf("rank: %d Writing sample data...\n", my_rank); */ +/* if ((ret = PIOc_setframe(ncid, varid, t))) */ +/* ERR(ret); */ +/* if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) */ +/* ERR(ret); */ +/* } */ + +/* /\* THis will cause all data to be written to disk. *\/ */ +/* if ((ret = PIOc_sync(ncid))) */ +/* ERR(ret); */ + +/* /\* Close the netCDF file. *\/ */ +/* printf("rank: %d Closing the sample data file...\n", my_rank); */ +/* if ((ret = PIOc_closefile(ncid))) */ +/* ERR(ret); */ + +/* /\* Check the output file. *\/ */ +/* /\* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, *\/ */ +/* /\* my_rank, ioid))) *\/ */ +/* /\* ERR(ret); *\/ */ +/* } */ + + /* Free the PIO decomposition. */ + /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */ + /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */ + /* ERR(ret); */ + + /* Finalize the IO system. Only call this from the computation tasks. */ + printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + } /* endif comp_task */ + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + printf("rank: %d SUCCESS!\n", my_rank); + return 0; + } diff --git a/src/externals/pio2/examples/c/darray_no_async.c b/src/externals/pio2/examples/c/darray_no_async.c new file mode 100644 index 000000000000..14228ab422c6 --- /dev/null +++ b/src/externals/pio2/examples/c/darray_no_async.c @@ -0,0 +1,358 @@ +/* + * @file + * @brief A simple C example for the ParallelIO Library. + * + * This example creates a netCDF output file with three dimensions + * (one unlimited) and one variable. It first writes, then reads the + * sample file using distributed arrays. + * + * This example can be run in parallel for 4 processors. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef TIMING +#include +#endif + +/* The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/* The number of dimensions in the example data. */ +#define NDIM3 3 + +/* The number of timesteps of data. */ +#define NUM_TIMESTEPS 2 + +/* The length of our sample data in X dimension.*/ +#define DIM_LEN_X 4 + +/* The length of our sample data in Y dimension.*/ +#define DIM_LEN_Y 4 + +/* The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/* Return code when netCDF output file does not match + * expectations. */ +#define ERR_BAD 1001 + +/* The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/* Number of tasks this example runs on. */ +#define TARGET_NTASKS 4 + +/* Logging level. */ +#define LOG_LEVEL 3 + +/* Lengths of dimensions. */ +int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + +/* Names of dimensions. */ +char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; + +/* Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/* Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/* Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/* This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/* @brief Check the output file. + * + * Use netCDF to check that the output is as expected. + * + * @param ntasks The number of processors running the example. + * @param filename The name of the example file to check. + * + * @return 0 if example file is correct, non-zero otherwise. */ +int check_file(int iosysid, int ntasks, char *filename, int iotype, + int elements_per_pe, int my_rank, int ioid) +{ + + int ncid; /* File ID from netCDF. */ + int ndims; /* Number of dimensions. */ + int nvars; /* Number of variables. */ + int ngatts; /* Number of global attributes. */ + int unlimdimid; /* ID of unlimited dimension. */ + int natts; /* Number of variable attributes. */ + nc_type xtype; /* NetCDF data type of this variable. */ + int ret; /* Return code for function calls. */ + int dimids[NDIM3]; /* Dimension ids for this variable. */ + char var_name[NC_MAX_NAME]; /* Name of the variable. */ + /* size_t start[NDIM3]; /\* Zero-based index to start read. *\/ */ + /* size_t count[NDIM3]; /\* Number of elements to read. *\/ */ + /* int buffer[DIM_LEN_X]; /\* Buffer to read in data. *\/ */ + /* int expected[DIM_LEN_X]; /\* Data values we expect to find. *\/ */ + + /* Open the file. */ + if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) + return ret; + printf("opened file %s ncid = %d\n", filename, ncid); + + /* Check the metadata. */ + if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) + return ret; + + /* Check the dimensions. */ + if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) + return ERR_BAD; + for (int d = 0; d < NDIM3; d++) + { + char my_dim_name[NC_MAX_NAME]; + PIO_Offset dimlen; + + if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) + return ret; + if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) + return ERR_BAD; + } + + /* Check the variable. */ + if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) + return ret; + if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || + dimids[2] != 2 || natts != 0) + return ERR_BAD; + + /* Allocate storage for sample data. */ + int buffer[elements_per_pe]; + int buffer_in[elements_per_pe]; + + /* Check each timestep. */ + for (int t = 0; t < NUM_TIMESTEPS; t++) + { + int varid = 0; /* There's only one var in sample file. */ + + /* This is the data we expect for this timestep. */ + for (int i = 0; i < elements_per_pe; i++) + buffer[i] = 100 * t + START_DATA_VAL + my_rank; + + /* Read one record. */ + if ((ret = PIOc_setframe(ncid, varid, t))) + ERR(ret); + if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) + return ret; + + /* Check the results. */ + for (int i = 0; i < elements_per_pe; i++) + if (buffer_in[i] != buffer[i]) + return ERR_BAD; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + /* Everything looks good! */ + return 0; +} + +/* Write, then read, a simple example with darrays. + + The sample file created by this program is a small netCDF file. It + has the following contents (as shown by ncdump): + + 
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+
+ +*/ + int main(int argc, char* argv[]) + { + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Rank of first task to be used for I/O. */ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + int iosysid; /* The ID for the parallel I/O system. */ + int ncid; /* The ncid of the netCDF file. */ + int dimid[NDIM3]; /* The dimension ID. */ + int varid; /* The ID of the netCDF varable. */ + int ioid; /* The I/O description ID. */ + char filename[NC_MAX_NAME + 1]; /* Test filename. */ + int num_flavors = 0; /* Number of iotypes available in this build. */ + int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */ + int ret; /* Return value. */ + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (ntasks != TARGET_NTASKS) + fprintf(stderr, "Number of processors must be 4!\n"); + printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", + my_rank, ntasks); + + /* Turn on logging. */ + if ((ret = PIOc_set_log_level(LOG_LEVEL))) + return ret; + + /* Initialize the PIO IO system. This specifies how many and + * which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, + ioproc_start, PIO_REARR_BOX, &iosysid))) + ERR(ret); + + /* Describe the decomposition. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS; + + /* Allocate and initialize array of decomposition mapping. */ + PIO_Offset compdof[elements_per_pe]; + for (int i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i; + + /* Create the PIO decomposition for this example. Since this + * is a variable with an unlimited dimension, we want to + * create a 2-D composition which represents one record. */ + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, + compdof, &ioid, 0, NULL, NULL))) + ERR(ret); + + /* The number of favors may change with the build parameters. */ +#ifdef _PNETCDF + format[num_flavors++] = PIO_IOTYPE_PNETCDF; +#endif + format[num_flavors++] = PIO_IOTYPE_NETCDF; +#ifdef _NETCDF4 + format[num_flavors++] = PIO_IOTYPE_NETCDF4C; + format[num_flavors++] = PIO_IOTYPE_NETCDF4P; +#endif + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* Create a filename. */ + sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename, format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimension and variable. */ + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM3; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) + ERR(ret); + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) + ERR(ret); + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Allocate storage for sample data. */ + int buffer[elements_per_pe]; + + /* Write each timestep. */ + for (int t = 0; t < NUM_TIMESTEPS; t++) + { + /* Create some data for this timestep. */ + for (int i = 0; i < elements_per_pe; i++) + buffer[i] = 100 * t + START_DATA_VAL + my_rank; + + /* Write data to the file. */ + printf("rank: %d Writing sample data...\n", my_rank); + if ((ret = PIOc_setframe(ncid, varid, t))) + ERR(ret); + if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) + ERR(ret); + } + + /* THis will cause all data to be written to disk. */ + if ((ret = PIOc_sync(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Check the output file. */ + /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */ + /* my_rank, ioid))) */ + /* ERR(ret); */ + } + + /* Free the PIO decomposition. */ + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + printf("rank: %d SUCCESS!\n", my_rank); + return 0; + } diff --git a/src/externals/pio2/examples/c/example1.c b/src/externals/pio2/examples/c/example1.c index 1fcd8ab222a2..ad650f0b1faf 100644 --- a/src/externals/pio2/examples/c/example1.c +++ b/src/externals/pio2/examples/c/example1.c @@ -285,7 +285,7 @@ int check_file(int ntasks, char *filename) { /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); - if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) MPIERR(ret); /* Learn my rank and the total number of processors. */ diff --git a/src/externals/pio2/src/clib/CMakeLists.txt b/src/externals/pio2/src/clib/CMakeLists.txt index d2c1c3bbeb68..7fac8891a8e8 100644 --- a/src/externals/pio2/src/clib/CMakeLists.txt +++ b/src/externals/pio2/src/clib/CMakeLists.txt @@ -54,24 +54,19 @@ install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include) #============================================================================== # DEFINE THE DEPENDENCIES #============================================================================== - -#===== MPI ===== -if (PIO_USE_MPISERIAL) - find_package (MPISERIAL COMPONENTS C REQUIRED) - if (MPISERIAL_C_FOUND) +# MPI test done at top level +if (MPISERIAL_C_FOUND) target_compile_definitions (pioc PRIVATE MPI_SERIAL) target_include_directories (pioc PUBLIC ${MPISERIAL_C_INCLUDE_DIRS}) target_link_libraries (pioc PUBLIC ${MPISERIAL_C_LIBRARIES}) - set (WITH_PNETCDF FALSE) - endif () -else () - find_package (MPI REQUIRED) endif () + + #===== GPTL ===== if (PIO_ENABLE_TIMING) find_package (GPTL COMPONENTS C QUIET) diff --git a/src/externals/pio2/src/clib/bget.c b/src/externals/pio2/src/clib/bget.c index 726ce5580da6..98bfc36893d2 100644 --- a/src/externals/pio2/src/clib/bget.c +++ b/src/externals/pio2/src/clib/bget.c @@ -400,7 +400,7 @@ #include #include #include -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC #include #endif @@ -614,7 +614,7 @@ bufsize requested_size; int compactseq = 0; #endif -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC // if(requested_size>maxsize){ // maxsize=requested_size; // printf("%s %d %d\n",__FILE__,__LINE__,maxsize); @@ -847,7 +847,7 @@ bufsize size; bufsize osize; /* Old size of buffer */ struct bhead *b; -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC return(realloc(buf, size)); #endif if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */ @@ -882,7 +882,7 @@ void *buf; { struct bfhead *b, *bn; -#ifdef PIO_USE_MALLOC +#if PIO_USE_MALLOC // printf("bget free %d %x\n",__LINE__,buf); free(buf); return; diff --git a/src/externals/pio2/src/clib/config.h.in b/src/externals/pio2/src/clib/config.h.in index 1722872c3056..8d80d37fd510 100644 --- a/src/externals/pio2/src/clib/config.h.in +++ b/src/externals/pio2/src/clib/config.h.in @@ -22,4 +22,7 @@ /** Set to non-zero to turn on logging. Output may be large. */ #define PIO_ENABLE_LOGGING @ENABLE_LOGGING@ +/** Size of MPI_Offset type. */ +#define SIZEOF_MPI_OFFSET @SIZEOF_MPI_OFFSET@ + #endif /* _PIO_CONFIG_ */ diff --git a/src/externals/pio2/src/clib/pio.h b/src/externals/pio2/src/clib/pio.h index fe234a155930..9dace3782c99 100644 --- a/src/externals/pio2/src/clib/pio.h +++ b/src/externals/pio2/src/clib/pio.h @@ -40,6 +40,9 @@ /** The maximum number of variables allowed in a netCDF file. */ #define PIO_MAX_VARS NC_MAX_VARS +/** The maximum number of dimensions allowed in a netCDF file. */ +#define PIO_MAX_DIMS NC_MAX_DIMS + /** Pass this to PIOc_set_iosystem_error_handling() as the iosysid in * order to set default error handling. */ #define PIO_DEFAULT (-1) @@ -115,7 +118,20 @@ typedef struct var_desc_t /** Number of requests bending with pnetcdf. */ int nreqs; - /** Buffer that contains the fill value for this variable. */ + /* Holds the fill value of this var. */ + void *fillvalue; + + /* The PIO data type (PIO_INT, PIO_FLOAT, etc.) */ + int pio_type; + + /* The size of the data type (2 for PIO_SHORT, 4 for PIO_INT, etc.) */ + PIO_Offset type_size; + + /** Non-zero if fill mode is turned on for this var. */ + int use_fill; + + /** Buffer that contains the holegrid fill values used to fill in + * missing sections of data when using the subset rearranger. */ void *fillbuf; /** Data buffer for this variable. */ @@ -191,10 +207,10 @@ enum PIO_REARR_COMM_FC_DIR typedef struct rearr_comm_fc_opt { /** Enable handshake */ - bool enable_hs; + bool hs; /** Enable isends - if false use blocking sends */ - bool enable_isend; + bool isend; /** Max pending requests * (PIO_REARR_COMM_UNLIMITED_PEND_REQ => unlimited pend req). @@ -217,10 +233,10 @@ typedef struct rearr_opt int fcd; /** flow control opts, comp to io procs */ - rearr_comm_fc_opt_t comm_fc_opts_comp2io; + rearr_comm_fc_opt_t comp2io; /** flow control opts, io to comp procs */ - rearr_comm_fc_opt_t comm_fc_opts_io2comp; + rearr_comm_fc_opt_t io2comp; } rearr_opt_t; /** @@ -245,10 +261,7 @@ typedef struct io_desc_t * io tasks. */ int nrecvs; - /** Local size of the decomposition array on the compute node. On - each compute task the application passes a compmap array of - length ndof. This array describes the arrangement of data in - memory on that task. */ + /** Local size of the decomposition array on the compute node. */ int ndof; /** All vars included in this io_desc_t have the same number of @@ -277,6 +290,9 @@ typedef struct io_desc_t /** The MPI type of the data. */ MPI_Datatype basetype; + /** The size in bytes of a datum of MPI type basetype. */ + int basetype_size; + /** Length of the iobuffer on this task for a single field on the * IO node. The arrays from compute nodes gathered and rearranged * to the io-nodes (which are sometimes collocated with compute @@ -287,24 +303,25 @@ typedef struct io_desc_t /** Maximum llen participating. */ int maxiobuflen; - /** Array of tasks received from in pio_swapm(). */ + /** Array (length nrecvs) of computation tasks received from. */ int *rfrom; - /** Array of counts of data to be received from each task in - * pio_swapm(). */ + /** Array (length nrecvs) of counts of data to be received from + * each computation task by the IO tasks. */ int *rcount; - /** Array of data count to send to each task in the communication - * in pio_swapm(). */ + /** Array (length numiotasks) of data counts to send to each task + * in the communication in pio_swapm(). */ int *scount; - /** Send index. */ + /** Array (length ndof) for the BOX rearranger with the index + * for computation taks (send side during writes). */ PIO_Offset *sindex; - /** Receive index. */ + /** Index for the IO tasks (receive side during writes). */ PIO_Offset *rindex; - /** Array of receive MPI types in pio_swapm() call. */ + /** Array (of length nrecvs) of receive MPI types in pio_swapm() call. */ MPI_Datatype *rtype; /** Array of send MPI types in pio_swapm() call. */ @@ -387,6 +404,11 @@ typedef struct iosystem_desc_t /** The number of tasks in the computation communicator. */ int num_comptasks; + /** The number of tasks in the union communicator (will be + * num_comptasks for non-async, num_comptasks + num_iotasks for + * async). */ + int num_uniontasks; + /** Rank of this task in the union communicator. */ int union_rank; @@ -419,6 +441,10 @@ typedef struct iosystem_desc_t * communicator. */ int *ioranks; + /** An array of the ranks of all computation tasks within the + * union communicator. */ + int *compranks; + /** Controls handling errors. */ int error_handler; @@ -427,11 +453,15 @@ typedef struct iosystem_desc_t int default_rearranger; /** True if asynchronous interface is in use. */ - bool async_interface; + bool async; /** True if this task is a member of the IO communicator. */ bool ioproc; + /** True if this task is a member of a computation + * communicator. */ + bool compproc; + /** MPI Info object. */ MPI_Info info; @@ -455,8 +485,13 @@ typedef struct wmulti_buffer * PIOc_Init_Decomp(). */ int ioid; - /** Number of variables in this multi variable buffer. */ - int validvars; + /** Non-zero if this is a buffer for a record var. */ + int recordvar; + + /** Number of arrays of data in the multibuffer. Each array had + * data for one var or record. When multibuffer is flushed, all + * arrays are written and num_arrays returns to zero. */ + int num_arrays; /** Size of this variables data on local task. All vars in the * multi-buffer have the same size. */ @@ -673,10 +708,19 @@ enum PIO_ERROR_HANDLERS #define PIO_EBADCHUNK NC_EBADCHUNK #define PIO_ENOTBUILT NC_ENOTBUILT #define PIO_EDISKLESS NC_EDISKLESS -#define PIO_FILL_DOUBLE NC_FILL_DOUBLE -#define PIO_FILL_FLOAT NC_FILL_FLOAT -#define PIO_FILL_INT NC_FILL_INT + +/* These are the netCDF default fill values. */ +#define PIO_FILL_BYTE NC_FILL_BYTE #define PIO_FILL_CHAR NC_FILL_CHAR +#define PIO_FILL_SHORT NC_FILL_SHORT +#define PIO_FILL_INT NC_FILL_INT +#define PIO_FILL_FLOAT NC_FILL_FLOAT +#define PIO_FILL_DOUBLE NC_FILL_DOUBLE +#define PIO_FILL_UBYTE NC_FILL_UBYTE +#define PIO_FILL_USHORT NC_FILL_USHORT +#define PIO_FILL_UINT NC_FILL_UINT +#define PIO_FILL_INT64 NC_FILL_INT64 +#define PIO_FILL_UINT64 NC_FILL_UINT64 #endif /* defined( _PNETCDF) || defined(_NETCDF) */ /** Define the extra error codes for the parallel-netcdf library. */ @@ -703,18 +747,20 @@ extern "C" { /* Decomposition. */ /* Init decomposition with 1-based compmap array. */ - int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, + int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, const PIO_Offset *iostart, const PIO_Offset *iocount); - int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, + int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen, const long int *start, const long int *count, int *ioidp); /* Init decomposition with 0-based compmap array. */ - int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, - const PIO_Offset *compmap, int *ioidp, const int *rearranger, + int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, + const PIO_Offset *compmap, int *ioidp, int rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount); - + + /* Free resources associated with a decomposition. */ int PIOc_freedecomp(int iosysid, int ioid); + int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen, PIO_Offset **map, MPI_Comm comm); int PIOc_readmap_from_f90(const char *file,int *ndims, int **gdims, PIO_Offset *maplen, @@ -729,16 +775,17 @@ extern "C" { /* Write a decomposition file using netCDF. */ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, - MPI_Comm comm, char *title, char *history, int fortran_order); + char *title, char *history, int fortran_order); /* Read a netCDF decomposition file. */ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioid, MPI_Comm comm, int pio_type, char *title, char *history, int *fortran_order); - /* Initializing IO system. */ - int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, + /* Initializing IO system for async. */ + int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm, - int *iosysidp); + int rearranger, int *iosysidp); + int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms, MPI_Comm io_comm, int *iosysidp); int PIOc_get_numiotasks(int iosysid, int *numiotasks); diff --git a/src/externals/pio2/src/clib/pio_darray.c b/src/externals/pio2/src/clib/pio_darray.c index 4250d87ea38e..1d6f15c7d308 100644 --- a/src/externals/pio2/src/clib/pio_darray.c +++ b/src/externals/pio2/src/clib/pio_darray.c @@ -49,6 +49,27 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit) * caller to use their own data buffering (instead of using the * buffering implemented in PIOc_write_darray()). * + * When the user calls PIOc_write_darray() one or more times, then + * PIO_write_darray_multi() will be called when the buffer is flushed. + * + * Internally, this function will: + *
    + *
  • Find info about file, decomposition, and variable. + *
  • Do a special flush for pnetcdf if needed. + *
  • Allocates a buffer big enough to hold all the data in the + * multi-buffer, for all tasks. + *
  • Calls rearrange_comp2io() to move data from compute to IO + * tasks. + *
  • For parallel iotypes (pnetcdf and netCDF-4 parallel) call + * pio_write_darray_multi_nc(). + *
  • For serial iotypes (netcdf classic and netCDF-4 serial) call + * write_darray_multi_serial(). + *
  • For subset rearranger, create holegrid to write missing + * data. Then call pio_write_darray_multi_nc() or + * write_darray_multi_serial() to write the holegrid. + *
  • Special buffer flush for pnetcdf. + *
+ * * @param ncid identifies the netCDF file. * @param varids an array of length nvars containing the variable ids to * be written. @@ -65,25 +86,23 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit) * that is on this processor. There are nvars arrays of data, and each * array of data contains one record worth of data for that variable. * @param frame an array of length nvars with the frame or record - * dimension for each of the nvars variables in IOBUF - * @param fillvalue pointer to the fill value to be used for missing - * data. Ignored if NULL. If provided, must be the correct fill value - * for the variable. The correct fill value will be used if NULL is - * passed. + * dimension for each of the nvars variables in IOBUF. NULL if this + * iodesc contains non-record vars. + * @param fillvalue pointer an array (of length nvars) of pointers to + * the fill value to be used for missing data. * @param flushtodisk non-zero to cause buffers to be flushed to disk. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ -int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen, - void *array, const int *frame, void **fillvalue, bool flushtodisk) +int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, + PIO_Offset arraylen, void *array, const int *frame, + void **fillvalue, bool flushtodisk) { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ io_desc_t *iodesc; /* Pointer to IO description information. */ - int vsize; /* size in bytes of the given data. */ int rlen; /* total data buffer size. */ var_desc_t *vdesc0; /* pointer to var_desc structure for each var. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code. */ /* Get the file info. */ @@ -98,61 +117,69 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI if (varids[v] < 0 || varids[v] > PIO_MAX_VARS) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld flushtodisk = %d", + LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld " + "flushtodisk = %d", ncid, ioid, nvars, arraylen, flushtodisk)); /* Check that we can write to this file. */ - if (! (file->mode & PIO_WRITE)) + if (!(file->mode & PIO_WRITE)) return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__); /* Get iodesc. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET, + "unknown rearranger", __FILE__, __LINE__); + + /* Get a pointer to the variable info for the first variable. */ + vdesc0 = &file->varlist[varids[0]]; + + /* if the buffer is already in use in pnetcdf we need to flush first */ + if (file->iotype == PIO_IOTYPE_PNETCDF && vdesc0->iobuf) + flush_output_buffer(file, 1, 0); - /* For netcdf serial writes we collect the data on io nodes and + pioassert(!vdesc0->iobuf, "buffer overwrite",__FILE__, __LINE__); + + /* Determine total size of aggregated data (all vars/records). + * For netcdf serial writes we collect the data on io nodes and * then move that data one node at a time to the io master node * and write (or read). The buffer size on io task 0 must be as * large as the largest used to accommodate this serial io - * method. */ - vdesc0 = file->varlist + varids[0]; - pioassert(!vdesc0->iobuf, "Attempt to overwrite existing io buffer",__FILE__, __LINE__); - - /* ??? */ - /* rlen = iodesc->llen*nvars; */ + * method. */ rlen = 0; if (iodesc->llen > 0) rlen = iodesc->maxiobuflen * nvars; - /* Currently there are two rearrangers box=1 and subset=2. There - * is never a case where rearranger==0. */ - LOG((2, "iodesc->rearranger = %d iodesc->needsfill = %d\n", iodesc->rearranger, - iodesc->needsfill)); - if (iodesc->rearranger > 0) + /* Allocate iobuf. */ + if (rlen > 0) { - if (rlen > 0) - { - if ((mpierr = MPI_Type_size(iodesc->basetype, &vsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "vsize = %d", vsize)); - - /* Allocate memory for the variable buffer. */ - if (!(vdesc0->iobuf = bget((size_t)vsize * (size_t)rlen))) - piomemerror(ios, (size_t)rlen * (size_t)vsize, __FILE__, __LINE__); - LOG((3, "allocated %ld bytes for variable buffer", (size_t)rlen * (size_t)vsize)); - - /* If data are missing for the BOX rearranger, insert fill values. */ - if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX) - for (int nv = 0; nv < nvars; nv++) - for (int i = 0; i < iodesc->maxiobuflen; i++) - memcpy(&((char *)vdesc0->iobuf)[vsize * (i + nv * iodesc->maxiobuflen)], - &((char *)fillvalue)[nv * vsize], vsize); - } - - /* Move data from compute to IO tasks. */ - if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Allocate memory for the buffer for all vars/records. */ + if (!(vdesc0->iobuf = bget(iodesc->basetype_size * (size_t)rlen))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated %lld bytes for variable buffer", (size_t)rlen * iodesc->basetype_size)); + + /* If fill values are desired, and we're using the BOX + * rearranger, insert fill values. */ + if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX) + for (int nv = 0; nv < nvars; nv++) + for (int i = 0; i < iodesc->maxiobuflen; i++) + memcpy(&((char *)vdesc0->iobuf)[iodesc->basetype_size * (i + nv * iodesc->maxiobuflen)], + &((char *)fillvalue)[nv * iodesc->basetype_size], iodesc->basetype_size); } - + else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc) + { + /* this assures that iobuf is allocated on all iotasks thus + assuring that the flush_output_buffer call above is called + collectively (from all iotasks) */ + if (!(vdesc0->iobuf = bget(1))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated token for variable buffer")); + } + + /* Move data from compute to IO tasks. */ + if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Write the darray based on the iotype. */ LOG((2, "about to write darray for iotype = %d", file->iotype)); switch (file->iotype) @@ -166,9 +193,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->num_aiotasks, vdesc0->iobuf, frame))) + if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 0, frame))) return pio_err(ios, file, ierr, __FILE__, __LINE__); break; @@ -182,6 +207,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI /* Release resources. */ if (vdesc0->iobuf) { + LOG((3,"freeing variable buffer in pio_darray")); brel(vdesc0->iobuf); vdesc0->iobuf = NULL; } @@ -202,22 +228,21 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars, iodesc->holegridsize, iodesc->needsfill)); - if (vdesc0->fillbuf) - piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); + pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__); /* Get a buffer. */ if (ios->io_rank == 0) - vdesc0->fillbuf = bget(iodesc->maxholegridsize * vsize * nvars); + vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->basetype_size * nvars); else if (iodesc->holegridsize > 0) - vdesc0->fillbuf = bget(iodesc->holegridsize * vsize * nvars); + vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->basetype_size * nvars); /* copying the fill value into the data buffer for the box * rearranger. This will be overwritten with data where * provided. */ for (int nv = 0; nv < nvars; nv++) for (int i = 0; i < iodesc->holegridsize; i++) - memcpy(&((char *)vdesc0->fillbuf)[vsize * (i + nv * iodesc->holegridsize)], - &((char *)fillvalue)[vsize * nv], vsize); + memcpy(&((char *)vdesc0->fillbuf)[iodesc->basetype_size * (i + nv * iodesc->holegridsize)], + &((char *)fillvalue)[iodesc->basetype_size * nv], iodesc->basetype_size); /* Write the darray based on the iotype. */ switch (file->iotype) @@ -232,10 +257,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - if ((ierr = pio_write_darray_multi_nc_serial(file, nvars, varids, iodesc->ndims, iodesc->basetype, - iodesc->maxfillregions, iodesc->fillregion, - iodesc->holegridsize, - iodesc->num_aiotasks, vdesc0->fillbuf, frame))) + if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 1, frame))) return pio_err(ios, file, ierr, __FILE__, __LINE__); break; default: @@ -254,7 +276,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI } } - /* Flush data to disk. */ + /* Flush data to disk for pnetcdf. */ if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF) if ((ierr = flush_output_buffer(file, flushtodisk, 0))) return pio_err(ios, file, ierr, __FILE__, __LINE__); @@ -262,6 +284,50 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI return PIO_NOERR; } +/** + * Find the fillvalue that should be used for a variable. + * + * @param file Info about file we are writing to. + * @param varid the variable ID. + * @param vdesc pointer to var_desc_t info for this var. + * @returns 0 for success, non-zero error code for failure. + * @ingroup PIO_write_darray + */ +int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + int no_fill; + int ierr; + + /* Check inputs. */ + pioassert(file && file->iosystem && vdesc, "invalid input", __FILE__, __LINE__); + ios = file->iosystem; + + LOG((3, "find_var_fillvalue file->pio_ncid = %d varid = %d", file->pio_ncid, varid)); + + /* Find out PIO data type of var. */ + if ((ierr = PIOc_inq_vartype(file->pio_ncid, varid, &vdesc->pio_type))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + + /* Find out length of type. */ + if ((ierr = PIOc_inq_type(file->pio_ncid, vdesc->pio_type, NULL, &vdesc->type_size))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + LOG((3, "getting fill value for varid = %d pio_type = %d type_size = %d", + varid, vdesc->pio_type, vdesc->type_size)); + + /* Allocate storage for the fill value. */ + if (!(vdesc->fillvalue = malloc(vdesc->type_size))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Get the fill value. */ + if ((ierr = PIOc_inq_var_fill(file->pio_ncid, varid, &no_fill, vdesc->fillvalue))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + vdesc->use_fill = no_fill ? 0 : 1; + LOG((3, "vdesc->use_fill = %d", vdesc->use_fill)); + + return PIO_NOERR; +} + /** * Write a distributed array to the output file. * @@ -269,20 +335,43 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PI * it to the IO nodes when the compute buffer is full or when a flush * is triggered. * + * Internally, this function will: + *
    + *
  • Locate info about this file, decomposition, and variable. + *
  • If we don't have a fillvalue for this variable, determine one + * and remember it for future calls. + *
  • Initialize or find the multi_buffer for this record/var. + *
  • Find out how much free space is available in the multi buffer + * and flush if needed. + *
  • Store the new user data in the mutli buffer. + *
  • If needed (only for subset rearranger), fill in gaps in data + * with fillvalue. + *
  • Remember the frame value (i.e. record number) of this data if + * there is one. + *
+ * + * NOTE: The write multi buffer wmulti_buffer is the cache on compute + * nodes that will collect and store multiple variables before sending + * them to the io nodes. Aggregating variables in this way leads to a + * considerable savings in communication expense. Variables in the wmb + * array must have the same decomposition and base data size and we + * also need to keep track of whether each is a recordvar (has an + * unlimited dimension) or not. + * * @param ncid the ncid of the open netCDF file. * @param varid the ID of the variable that these data will be written * to. * @param ioid the I/O description ID as passed back by * PIOc_InitDecomp(). - * @param arraylen the length of the array to be written. This should + * @param arraylen the length of the array to be written. This should * be at least the length of the local component of the distrubited * array. (Any values beyond length of the local component will be * ignored.) * @param array pointer to an array of length arraylen with the data * to be written. This is a pointer to the distributed portion of the * array that is on this task. - * @param fillvalue pointer to the fill value to be used for - * missing data. + * @param fillvalue pointer to the fill value to be used for missing + * data. * @returns 0 for success, non-zero error code for failure. * @ingroup PIO_write_darray */ @@ -296,8 +385,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * void *bufptr; /* A data buffer. */ MPI_Datatype vtype; /* The MPI type of the variable. */ wmulti_buffer *wmb; /* The write multi buffer for one or more vars. */ - int tsize; /* Size of MPI type. */ - bool recordvar; /* True if this is a record variable. */ + int recordvar; /* Non-zero if this is a record variable. */ int needsflush = 0; /* True if we need to flush buffer. */ bufsize totfree; /* Amount of free space in the buffer. */ bufsize maxfree; /* Max amount of free space in buffer. */ @@ -320,95 +408,59 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); - /* Get var description. */ - vdesc = file->varlist + varid; - LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims, vdesc->nreqs)); - - /* Is this a record variable? */ - recordvar = vdesc->record >= 0 ? true : false; - LOG((3, "recordvar = %d", recordvar)); - /* Check that the local size of the variable passed in matches the - * size expected by the io descriptor. */ + * size expected by the io descriptor. Fail if arraylen is too + * small, just put a warning in the log if it is too big (the + * excess values will be ignored.) */ if (arraylen < iodesc->ndof) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); + LOG((2, "%s arraylen = %d iodesc->ndof = %d", + (iodesc->ndof != arraylen) ? "WARNING: iodesc->ndof != arraylen" : "", + arraylen, iodesc->ndof)); - if (iodesc->ndof != arraylen) - LOG((1, "User supplied array is larger than expected, arraylen != iodesc->ndof")); - - /* Get a pointer to the buffer space for this file. It will hold - * data from one or more variables that fit the same - * description. */ - wmb = &file->buffer; + /* Get var description. */ + vdesc = &(file->varlist[varid]); + LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims, + vdesc->nreqs)); + + /* If we don't know the fill value for this var, get it. */ + if (!vdesc->fillvalue) + if ((ierr = find_var_fillvalue(file, varid, vdesc))) + return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + + /* Is this a record variable? The user must set the vdesc->record + * value by calling PIOc_setframe() before calling this + * function. */ + recordvar = vdesc->record >= 0 ? 1 : 0; + LOG((3, "recordvar = %d", recordvar)); - /* If the ioid is not initialized, set it. For non record vars, - * use the negative ??? */ - if (wmb->ioid == -1) - wmb->ioid = recordvar ? ioid : -ioid; - else - { - /* Handle record and non-record variables differently. */ - if (recordvar) - { - /* Moving to the end of the wmb linked list to add the - * current variable. ??? */ - while(wmb->next && wmb->ioid != ioid) - if (wmb->next) - wmb = wmb->next; -#ifdef _PNETCDF - /* Do we still need the commented code below? ??? */ - /* flush the previous record before starting a new one. this is collective */ - /* if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || - (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ - needsflush = 2; // flush to disk - } */ -#endif - } - else - { - /* Move to end of list. */ - while(wmb->next && wmb->ioid != -(ioid)) - if (wmb->next) - wmb = wmb->next; - } - } + /* Move to end of list or the entry that matches this ioid. */ + for (wmb = &file->buffer; wmb->next; wmb = wmb->next) + if (wmb->ioid == ioid && wmb->recordvar == recordvar) + break; - /* The write multi buffer wmulti_buffer is the cache on compute - nodes that will collect and store multiple variables before - sending them to the io nodes. Aggregating variables in this way - leads to a considerable savings in communication - expense. Variables in the wmb array must have the same - decomposition and base data size and we also need to keep track - of whether each is a recordvar (has an unlimited dimension) or - not. */ - if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))) + /* If we did not find an existing wmb entry, create a new wmb. */ + if (wmb->ioid != ioid || wmb->recordvar != recordvar) { /* Allocate a buffer. */ if (!(wmb->next = bget((bufsize)sizeof(wmulti_buffer)))) - piomemerror(ios, sizeof(wmulti_buffer), __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); LOG((3, "allocated multi-buffer")); /* Set pointer to newly allocated buffer and initialize.*/ wmb = wmb->next; + wmb->recordvar = recordvar; wmb->next = NULL; - wmb->ioid = recordvar ? ioid : -ioid; - wmb->validvars = 0; + wmb->ioid = ioid; + wmb->num_arrays = 0; wmb->arraylen = arraylen; wmb->vid = NULL; wmb->data = NULL; wmb->frame = NULL; wmb->fillvalue = NULL; } - - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - LOG((2, "wmb->validvars = %d arraylen = %d tsize = %d\n", wmb->validvars, - arraylen, tsize)); - - /* At this point wmb should be pointing to a new or existing buffer - so we can add the data. */ + LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->basetype_size = %d\n", + wmb->num_arrays, arraylen, iodesc->basetype_size)); /* Find out how much free, contiguous space is available. */ bfreespace(&totfree, &maxfree); @@ -416,23 +468,24 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * /* maxfree is the available memory. If that is < 10% greater than * the size of the current request needsflush is true. */ if (needsflush == 0) - needsflush = (maxfree <= 1.1 * (1 + wmb->validvars) * arraylen * tsize); + needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size); /* Tell all tasks on the computation communicator whether we need * to flush data. */ - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm))) + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, + ios->comp_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); LOG((2, "needsflush = %d", needsflush)); /* Flush data if needed. */ if (needsflush > 0) { - LOG((2, "maxfree = %ld wmb->validvars = %d (1 + wmb->validvars) * arraylen * tsize = %ld totfree = %ld\n", - maxfree, wmb->validvars, (1 + wmb->validvars) * arraylen * tsize, totfree)); - #ifdef PIO_ENABLE_LOGGING /* Collect a debug report about buffer. */ cn_buffer_report(ios, true); + LOG((2, "maxfree = %ld wmb->num_arrays = %d (1 + wmb->num_arrays) *" + " arraylen * iodesc->basetype_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays, + (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size, totfree)); #endif /* PIO_ENABLE_LOGGING */ /* If needsflush == 2 flush to disk otherwise just flush to io node. */ @@ -443,91 +496,119 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * /* Get memory for data. */ if (arraylen > 0) { - if (!(wmb->data = bgetr(wmb->data, (1 + wmb->validvars) * arraylen * tsize))) - piomemerror(ios, (1 + wmb->validvars) * arraylen * tsize, __FILE__, __LINE__); - LOG((2, "got %ld bytes for data", (1 + wmb->validvars) * arraylen * tsize)); + if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size)); } /* vid is an array of variable ids in the wmb list, grow the list * and add the new entry. */ - if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__); + if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* wmb->frame is the record number, we assume that the variables * in the wmb list may not all have the same unlimited dimension * value although they usually do. */ if (vdesc->record >= 0) - if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__); + if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - /* If we need a fill value, get it. */ + /* If we need a fill value, get it. If we are using the subset + * rearranger and not using the netcdf fill mode then we need to + * do an extra write to fill in the holes with the fill value. */ if (iodesc->needsfill) { /* Get memory to hold fill value. */ - if (!(wmb->fillvalue = bgetr(wmb->fillvalue, tsize * (1 + wmb->validvars)))) - piomemerror(ios, (1 + wmb->validvars) * tsize, __FILE__, __LINE__); + if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->basetype_size * (1 + wmb->num_arrays)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* If the user passed a fill value, use that, otherwise use * the default fill value of the netCDF type. Copy the fill * value to the buffer. */ if (fillvalue) { - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, fillvalue, tsize); - LOG((3, "copied user-provided fill value tsize = %d", tsize)); + memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays, + fillvalue, iodesc->basetype_size); + LOG((3, "copied user-provided fill value iodesc->basetype_size = %d", + iodesc->basetype_size)); } else { + void *fill; + signed char byte_fill = PIO_FILL_BYTE; + char char_fill = PIO_FILL_CHAR; + short short_fill = PIO_FILL_SHORT; + int int_fill = PIO_FILL_INT; + float float_fill = PIO_FILL_FLOAT; + double double_fill = PIO_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = PIO_FILL_UBYTE; + unsigned short ushort_fill = PIO_FILL_USHORT; + unsigned int uint_fill = PIO_FILL_UINT; + long long int64_fill = PIO_FILL_INT64; + long long uint64_fill = PIO_FILL_UINT64; +#endif /* _NETCDF4 */ vtype = (MPI_Datatype)iodesc->basetype; LOG((3, "caller did not provide fill value vtype = %d", vtype)); - if (vtype == MPI_INT) - { - int fill = PIO_FILL_INT; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + + /* This must be done with an if statement, not a case, or + * openmpi will not build. */ + if (vtype == MPI_BYTE) + fill = &byte_fill; + else if (vtype == MPI_CHAR) + fill = &char_fill; + else if (vtype == MPI_SHORT) + fill = &short_fill; + else if (vtype == MPI_INT) + fill = &int_fill; else if (vtype == MPI_FLOAT) - { - float fill = PIO_FILL_FLOAT; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + fill = &float_fill; else if (vtype == MPI_DOUBLE) - { - double fill = PIO_FILL_DOUBLE; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } - else if (vtype == MPI_CHARACTER) - { - char fill = PIO_FILL_CHAR; - memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize); - } + fill = &double_fill; +#ifdef _NETCDF4 + else if (vtype == MPI_UNSIGNED_CHAR) + fill = &ubyte_fill; + else if (vtype == MPI_UNSIGNED_SHORT) + fill = &ushort_fill; + else if (vtype == MPI_UNSIGNED) + fill = &uint_fill; + else if (vtype == MPI_LONG_LONG) + fill = &int64_fill; + else if (vtype == MPI_UNSIGNED_LONG_LONG) + fill = &uint64_fill; +#endif /* _NETCDF4 */ else return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + + memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays, + fill, iodesc->basetype_size); + LOG((3, "copied fill value")); } } /* Tell the buffer about the data it is getting. */ wmb->arraylen = arraylen; - wmb->vid[wmb->validvars] = varid; + wmb->vid[wmb->num_arrays] = varid; + LOG((3, "wmb->num_arrays = %d wmb->vid[wmb->num_arrays] = %d", wmb->num_arrays, + wmb->vid[wmb->num_arrays])); /* Copy the user-provided data to the buffer. */ - bufptr = (void *)((char *)wmb->data + arraylen * tsize * wmb->validvars); + bufptr = (void *)((char *)wmb->data + arraylen * iodesc->basetype_size * wmb->num_arrays); if (arraylen > 0) { - memcpy(bufptr, array, arraylen * tsize); - LOG((3, "copied %ld bytes of user data", arraylen * tsize)); + memcpy(bufptr, array, arraylen * iodesc->basetype_size); + LOG((3, "copied %ld bytes of user data", arraylen * iodesc->basetype_size)); } /* Add the unlimited dimension value of this variable to the frame * array in wmb. */ if (wmb->frame) - wmb->frame[wmb->validvars] = vdesc->record; - wmb->validvars++; + wmb->frame[wmb->num_arrays] = vdesc->record; + wmb->num_arrays++; - LOG((2, "wmb->validvars = %d iodesc->maxbytes / tsize = %d iodesc->ndof = %d iodesc->llen = %d", - wmb->validvars, iodesc->maxbytes / tsize, iodesc->ndof, iodesc->llen)); - - /* Call the sync when ??? */ - if (wmb->validvars >= iodesc->maxbytes / tsize) - PIOc_sync(ncid); + LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->basetype_size = %d " + "iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays, + iodesc->maxbytes / iodesc->basetype_size, iodesc->ndof, iodesc->llen)); return PIO_NOERR; } @@ -556,8 +637,6 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, io_desc_t *iodesc; /* Pointer to IO description information. */ void *iobuf = NULL; /* holds the data as read on the io node. */ size_t rlen = 0; /* the length of data in iobuf. */ - int tsize; /* Total size. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code. */ /* Get the file info. */ @@ -568,6 +647,8 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, /* Get the iodesc. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET, + "unknown rearranger", __FILE__, __LINE__); /* ??? */ if (ios->iomaster == MPI_ROOT) @@ -575,24 +656,10 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, else rlen = iodesc->llen; - /* Is a rearranger in use? */ - if (iodesc->rearranger > 0) - { - if (ios->ioproc && rlen > 0) - { - /* Get the MPI type size. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Allocate a buffer for one record. */ - if (!(iobuf = bget((size_t)tsize * rlen))) - piomemerror(ios, rlen * (size_t)tsize, __FILE__, __LINE__); - } - } - else - { - iobuf = array; - } + /* Allocate a buffer for one record. */ + if (ios->ioproc && rlen > 0) + if (!(iobuf = bget(iodesc->basetype_size * rlen))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Call the correct darray read function based on iotype. */ switch (file->iotype) @@ -611,16 +678,13 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__); } - /* If a rearranger was specified, rearrange the data. */ - if (iodesc->rearranger > 0) - { - if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* Rearrange the data. */ + if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Free the buffer. */ - if (rlen > 0) - brel(iobuf); - } + /* Free the buffer. */ + if (rlen > 0) + brel(iobuf); return PIO_NOERR; } diff --git a/src/externals/pio2/src/clib/pio_darray_int.c b/src/externals/pio2/src/clib/pio_darray_int.c index ac19b37e198e..79572219b91d 100644 --- a/src/externals/pio2/src/clib/pio_darray_int.c +++ b/src/externals/pio2/src/clib/pio_darray_int.c @@ -25,6 +25,16 @@ extern void *CN_bpool; /* Maximum buffer usage. */ extern PIO_Offset maxusage; +/* handler for freeing the memory buffer pool */ +void bpool_free(void *p) +{ + free(p); + if(p == CN_bpool){ + CN_bpool = NULL; + } +} + + /** * Initialize the compute buffer to size pio_cnbuffer_limit. * @@ -49,10 +59,10 @@ int compute_buffer_init(iosystem_desc_t *ios) if (!CN_bpool) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - bectl(NULL, malloc, free, pio_cnbuffer_limit); + bectl(NULL, malloc, bpool_free, pio_cnbuffer_limit); } #endif - LOG((2, "compute_buffer_init CN_bpool = %d", CN_bpool)); + LOG((2, "compute_buffer_init complete")); return PIO_NOERR; } @@ -60,7 +70,7 @@ int compute_buffer_init(iosystem_desc_t *ios) /** * Write a set of one or more aggregated arrays to output file. This * function is only used with parallel-netcdf and netcdf-4 parallel - * iotypes. Serial io types use pio_write_darray_multi_nc_serial(). + * iotypes. Serial io types use write_darray_multi_serial(). * * @param file a pointer to the open file descriptor for the file * that will be written to @@ -85,7 +95,7 @@ int compute_buffer_init(iosystem_desc_t *ios) * less than blocksize*numiotasks then some iotasks will have a NULL * iobuf. * @param frame the frame or record dimension for each of the nvars - * variables in iobuf. + * variables in iobuf. NULL if this iodesc contains non-record vars. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ @@ -122,7 +132,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int vdesc = file->varlist + vid[0]; /* If async is in use, send message to IO master task. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -350,6 +360,298 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int return ierr; } +/** + * Fill the tmp_start and tmp_count arrays, which contain the start + * and count arrays for all regions. + * + * This is an internal function which is only called on io tasks. It + * is called by write_darray_multi_serial(). + * + * @param region pointer to the first in a linked list of regions. + * @param maxregions the number of regions in the list. + * @param fndims the number of dimensions in the file. + * @param iodesc_ndims the number of dimensions in the decomposition. + * @param vdesc pointer to an array of var_desc_t for the vars being + * written. + * @param tmp_start pointer to an already allocaed array of length + * fndims * maxregions. This array will get the start values for all + * regions. + * @param tmp_count pointer to an already allocaed array of length + * fndims * maxregions. This array will get the count values for all + * regions. + * @returns 0 for success, error code otherwise. + * @ingroup PIO_read_darray + **/ +int find_all_start_count(io_region *region, int maxregions, int fndims, + int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start, + size_t *tmp_count) +{ + /* Check inputs. */ + pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc && + tmp_start && tmp_count, "invalid input", __FILE__, __LINE__); + + /* Find the start/count arrays for each region in the list. */ + for (int r = 0; r < maxregions; r++) + { + /* Initialize the start/count arrays for this region to 0. */ + for (int d = 0; d < fndims; d++) + { + tmp_start[d + r * fndims] = 0; + tmp_count[d + r * fndims] = 0; + } + + if (region) + { + if (vdesc->record >= 0) + { + /* This is a record based multidimensional + * array. Copy start/count for non-record + * dimensions. */ + for (int i = fndims - iodesc_ndims; i < fndims; i++) + { + tmp_start[i + r * fndims] = region->start[i - (fndims - iodesc_ndims)]; + tmp_count[i + r * fndims] = region->count[i - (fndims - iodesc_ndims)]; + LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, + tmp_start[i + r * fndims], i + r * fndims, + tmp_count[i + r * fndims])); + } + } + else + { + /* This is not a record based multidimensional array. */ + for (int i = 0; i < iodesc_ndims; i++) + { + tmp_start[i + r * fndims] = region->start[i]; + tmp_count[i + r * fndims] = region->count[i]; + LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims, + tmp_start[i + r * fndims], i + r * fndims, + tmp_count[i + r * fndims])); + } + } + + /* Move to next region. */ + region = region->next; + + } /* endif region */ + } /* next r */ + + return PIO_NOERR; +} + +/** + * Internal function called by IO tasks other than IO task 0 to send + * their tmp_start/tmp_count arrays to IO task 0. + * + * This is an internal function which is only called on io tasks other + * than IO task 0. It is called by write_darray_multi_serial(). + * + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray + **/ +int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset llen, + int maxregions, int nvars, int fndims, size_t *tmp_start, + size_t *tmp_count, void *iobuf) +{ + MPI_Status status; /* Recv status for MPI. */ + int mpierr; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ + + /* Check inputs. */ + pioassert(ios && ios->ioproc && ios->io_rank > 0 && maxregions >= 0, + "invalid inputs", __FILE__, __LINE__); + + /* Do a handshake. */ + if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + + /* Send local length of iobuffer for each field (all + * fields are the same length). */ + if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "sent llen = %d", llen)); + + /* Send the number of data regions, the start/count for + * all regions, and the data buffer with all the data. */ + if (llen > 0) + { + if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks, + ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0, + ios->io_rank + 2 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0, + ios->io_rank + 3 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Send(iobuf, nvars * llen, iodesc->basetype, 0, + ios->io_rank + 4 * ios->num_iotasks, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "sent data for maxregions = %d", maxregions)); + } + + return PIO_NOERR; +} + +/** + * This is an internal function that is run only on IO proc 0. It + * receives data from all the other IO tasks, and write that data to + * disk. This is called from write_darray_multi_serial(). + * + * @param file a pointer to the open file descriptor for the file + * that will be written to. + * @param vid an array of the variable ids to be written + * @param frame the record dimension for each of the nvars variables + * in iobuf. NULL if this iodesc contains non-record vars. + * @param iodesc pointer to the decomposition info. + * @param llen length of the iobuffer on this task for a single + * field. + * @param maxregions max number of blocks to be written from this + * iotask. + * @param nvars the number of variables to be written with this + * decomposition. + * @param fndims the number of dimensions in the file. + * @param tmp_start pointer to an already allocaed array of length + * fndims * maxregions. This array will get the start values for all + * regions. + * @param tmp_count pointer to an already allocaed array of length + * fndims * maxregions. This array will get the count values for all + * regions. + * @param iobuf the buffer to be written from this mpi task. May be + * null. for example we have 8 ionodes and a distributed array with + * global size 4, then at least 4 nodes will have a null iobuf. In + * practice the box rearranger trys to have at least blocksize bytes + * on each io task and so if the total number of bytes to write is + * less than blocksize*numiotasks then some iotasks will have a NULL + * iobuf. + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray + */ +int recv_and_write_data(file_desc_t *file, const int *vid, const int *frame, + io_desc_t *iodesc, PIO_Offset llen, int maxregions, int nvars, + int fndims, size_t *tmp_start, size_t *tmp_count, void *iobuf) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + size_t rlen; /* Length of IO buffer on this task. */ + int rregions; /* Number of regions in buffer for this task. */ + size_t start[fndims], count[fndims]; + size_t loffset; + void *bufptr; + var_desc_t *vdesc; /* Contains info about the variable. */ + MPI_Status status; /* Recv status for MPI. */ + int mpierr; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ + + ios = file->iosystem; + + /* For each of the other tasks that are using this task + * for IO. */ + for (int rtask = 0; rtask < ios->num_iotasks; rtask++) + { + /* From the remote tasks, we send information about + * the data regions. and also the data. */ + if (rtask) + { + /* handshake - tell the sending task I'm ready */ + if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + + /* Get length of iobuffer for each field on this + * task (all fields are the same length). */ + if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, + &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "received rlen = %d", rlen)); + + /* Get the number of regions, the start/count + * values for all regions, and the data buffer. */ + if (rlen > 0) + { + if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks, + ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask, + rtask + 2 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask, + rtask + 3 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Recv(iobuf, nvars * rlen, iodesc->basetype, rtask, + rtask + 4 * ios->num_iotasks, ios->io_comm, &status))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "received data rregions = %d fndims = %d", rregions, fndims)); + } + } + else /* task 0 */ + { + rlen = llen; + rregions = maxregions; + } + LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions)); + + /* If there is data from this task, write it. */ + if (rlen > 0) + { + loffset = 0; + for (int regioncnt = 0; regioncnt < rregions; regioncnt++) + { + LOG((3, "writing data for region with regioncnt = %d", regioncnt)); + + /* Get the start/count arrays for this region. */ + for (int i = 0; i < fndims; i++) + { + start[i] = tmp_start[i + regioncnt * fndims]; + count[i] = tmp_count[i + regioncnt * fndims]; + LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i])); + } + + /* Process each variable in the buffer. */ + for (int nv = 0; nv < nvars; nv++) + { + LOG((3, "writing buffer var %d", nv)); + vdesc = file->varlist + vid[0]; + + /* Get a pointer to the correct part of the buffer. */ + bufptr = (void *)((char *)iobuf + iodesc->basetype_size * (nv * rlen + loffset)); + + /* If this var has an unlimited dim, set + * the start on that dim to the frame + * value for this variable. */ + if (vdesc->record >= 0) + { + if (fndims > 1 && iodesc->ndims < fndims && count[1] > 0) + { + count[0] = 1; + start[0] = frame[nv]; + } + else if (fndims == iodesc->ndims) + { + start[0] += vdesc->record; + } + } + + /* Call the netCDF functions to write the data. */ + if ((ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr))) + return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__); + + } /* next var */ + + /* Calculate the total size. */ + size_t tsize = 1; + for (int i = 0; i < fndims; i++) + tsize *= count[i]; + + /* Keep track of where we are in the buffer. */ + loffset += tsize; + + LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt, + tsize, loffset)); + } /* next regioncnt */ + } /* endif (rlen > 0) */ + } /* next rtask */ + + return PIO_NOERR; +} + /** * Write a set of one or more aggregated arrays to output file in * serial mode. This function is called for netCDF classic and @@ -361,16 +663,13 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int * @param nvars the number of variables to be written with this * decomposition. * @param vid an array of the variable ids to be written - * @param iodesc_ndims the number of dimensions explicitly in the - * iodesc. - * @param basetype the basic type of the minimal data unit + * @param iodesc pointer to the decomposition info. * @param maxregions max number of blocks to be written from this * iotask. * @param firstregion pointer to the first element of a linked * list of region descriptions. May be NULL. * @param llen length of the iobuffer on this task for a single * field. - * @param num_aiotasks actual number of iotasks participating * @param iobuf the buffer to be written from this mpi task. May be * null. for example we have 8 ionodes and a distributed array with * global size 4, then at least 4 nodes will have a null iobuf. In @@ -379,48 +678,47 @@ int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int * less than blocksize*numiotasks then some iotasks will have a NULL * iobuf. * @param frame the record dimension for each of the nvars variables - * in iobuf. + * in iobuf. NULL if this iodesc contains non-record vars. * @return 0 for success, error code otherwise. * @ingroup PIO_write_darray */ -int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims, - MPI_Datatype basetype, int maxregions, io_region *firstregion, - PIO_Offset llen, int num_aiotasks, void *iobuf, - const int *frame) +int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid, + io_desc_t *iodesc, int fill, const int *frame) { iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; /* Contains info about the variable. */ - int dsize; /* Size in bytes of one element of basetype. */ int fndims; /* Number of dims in the var in the file. */ - int tsize; /* Size of the MPI type, in bytes. */ - MPI_Status status; /* Recv status for MPI. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr; /* Return code. */ /* Check inputs. */ - pioassert(file && file->iosystem && vid && vid[0] >= 0 && vid[0] <= PIO_MAX_VARS, - "invalid input", __FILE__, __LINE__); + pioassert(file && file->iosystem && file->varlist && vid && vid[0] >= 0 && + vid[0] <= PIO_MAX_VARS && iodesc, "invalid input", __FILE__, __LINE__); - LOG((1, "pio_write_darray_multi_nc_serial nvars = %d iodesc_ndims = %d basetype = %d " - "maxregions = %d llen = %d num_aiotasks = %d", nvars, iodesc_ndims, - basetype, maxregions, llen, num_aiotasks)); - -#ifdef TIMING - /* Start timing this function. */ - GPTLstart("PIO:write_darray_multi_nc_serial"); -#endif + LOG((1, "write_darray_multi_serial nvars = %d iodesc->ndims = %d iodesc->basetype = %d", + nvars, iodesc->ndims, iodesc->basetype)); /* Get the iosystem info. */ ios = file->iosystem; /* Get the var info. */ vdesc = file->varlist + vid[0]; + LOG((2, "vdesc record %d ndims %d nreqs %d ios->async = %d", vdesc->record, + vdesc->ndims, vdesc->nreqs, ios->async)); - LOG((2, "vdesc record %d ndims %d nreqs %d ios->async_interface = %d", vdesc->record, - vdesc->ndims, vdesc->nreqs, ios->async_interface)); + /* Set these differently for data and fill writing. */ + int num_regions = fill ? iodesc->maxfillregions: iodesc->maxregions; + io_region *region = fill ? iodesc->fillregion : iodesc->firstregion; + PIO_Offset llen = fill ? iodesc->holegridsize : iodesc->llen; + void *iobuf = fill ? vdesc->fillbuf : vdesc->iobuf; + +#ifdef TIMING + /* Start timing this function. */ + GPTLstart("PIO:write_darray_multi_nc_serial"); +#endif /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -444,220 +742,36 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vi if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims))) return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Get the size of the type. */ - if ((mpierr = MPI_Type_size(basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "fndims = %d tsize = %d", fndims, tsize)); - /* Only IO tasks participate in this code. */ if (ios->ioproc) { - io_region *region; - void *bufptr; - size_t tmp_start[fndims * maxregions]; /* A start array for each region. */ - size_t tmp_count[fndims * maxregions]; /* A count array for each region. */ + size_t tmp_start[fndims * num_regions]; /* A start array for each region. */ + size_t tmp_count[fndims * num_regions]; /* A count array for each region. */ - LOG((3, "maxregions = %d", maxregions)); + LOG((3, "num_regions = %d", num_regions)); /* Fill the tmp_start and tmp_count arrays, which contain the * start and count arrays for all regions. */ - region = firstregion; - for (int regioncnt = 0; regioncnt < maxregions; regioncnt++) - { - /* Initialize the start/count arrays for this region to 0. */ - for (int i = 0; i < fndims; i++) - { - tmp_start[i + regioncnt * fndims] = 0; - tmp_count[i + regioncnt * fndims] = 0; - } - - if (region) - { - if (vdesc->record >= 0) - { - /* This is a record based multidimensional - * array. Copy start/count for non-record - * dimensions. */ - for (int i = fndims - iodesc_ndims; i < fndims; i++) - { - tmp_start[i + regioncnt * fndims] = region->start[i - (fndims - iodesc_ndims)]; - tmp_count[i + regioncnt * fndims] = region->count[i - (fndims - iodesc_ndims)]; - LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, - tmp_start[i + regioncnt * fndims], i + regioncnt * fndims, - tmp_count[i + regioncnt * fndims])); - } - } - else - { - /* This is not a record based multidimensional array. */ - for (int i = 0; i < iodesc_ndims; i++) - { - tmp_start[i + regioncnt * fndims] = region->start[i]; - tmp_count[i + regioncnt * fndims] = region->count[i]; - LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, - tmp_start[i + regioncnt * fndims], i + regioncnt * fndims, - tmp_count[i + regioncnt * fndims])); - } - } - - /* Move to next region. */ - region = region->next; - - } /* endif region */ - } /* next regioncnt */ + if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, vdesc, + tmp_start, tmp_count))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Tasks other than 0 will send their data to task 0. */ if (ios->io_rank > 0) { - /* Do a handshake. */ - if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Send local length of iobuffer for each field (all - * fields are the same length). */ - if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "sent llen = %d", llen)); - - /* Send the number of data regions, the start/count for - * all regions, and the data buffer with all the data. */ - if (llen > 0) - { - if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 2 * ios->num_iotasks, - ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0, ios->io_rank + 3 * ios->num_iotasks, - ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Send(iobuf, nvars * llen, basetype, 0, ios->io_rank + 4 * ios->num_iotasks, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "sent data for maxregions = %d", maxregions)); - } + /* Send the tmp_start and tmp_count arrays from this IO task + * to task 0. */ + if ((ierr = send_all_start_count(ios, iodesc, llen, num_regions, nvars, fndims, + tmp_start, tmp_count, iobuf))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } else { /* Task 0 will receive data from all other IO tasks. */ - size_t rlen; /* Length of IO buffer on this task. */ - int rregions; /* Number of regions in buffer for this task. */ - size_t start[fndims], count[fndims]; - size_t loffset; - - /* Get the size of the MPI data type. */ - if ((mpierr = MPI_Type_size(basetype, &dsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "dsize = %d", dsize)); - - /* For each of the other tasks that are using this task - * for IO. */ - for (int rtask = 0; rtask < ios->num_iotasks; rtask++) - { - /* From the remote tasks, we send information about - * the data regions. and also the data. */ - if (rtask) - { - /* handshake - tell the sending task I'm ready */ - if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Get length of iobuffer for each field on this - * task (all fields are the same length). */ - if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "received rlen = %d", rlen)); - /* Get the number of regions, the start/count - * values for all regions, and the data buffer. */ - if (rlen > 0) - { - if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask, rtask + 2 * ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask, rtask + 3 * ios->num_iotasks, - ios->io_comm, &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Recv(iobuf, nvars * rlen, basetype, rtask, rtask + 4 * ios->num_iotasks, ios->io_comm, - &status))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((3, "received data rregions = %d fndims = %d", rregions, fndims)); - } - } - else /* task 0 */ - { - rlen = llen; - rregions = maxregions; - } - LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions)); - - /* If there is data from this task, write it. */ - if (rlen > 0) - { - loffset = 0; - for (int regioncnt = 0; regioncnt < rregions; regioncnt++) - { - LOG((3, "writing data for region with regioncnt = %d", regioncnt)); - - /* Get the start/count arrays for this region. */ - for (int i = 0; i < fndims; i++) - { - start[i] = tmp_start[i + regioncnt * fndims]; - count[i] = tmp_count[i + regioncnt * fndims]; - LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i])); - } - - /* Process each variable in the buffer. */ - for (int nv = 0; nv < nvars; nv++) - { - LOG((3, "writing buffer var %d", nv)); - - /* Get a pointer to the correct part of the buffer. */ - bufptr = (void *)((char *)iobuf + tsize * (nv * rlen + loffset)); - - /* If this var has an unlimited dim, set - * the start on that dim to the frame - * value for this variable. */ - if (vdesc->record >= 0) - { - if (fndims > 1 && iodesc_ndims < fndims && count[1] > 0) - { - count[0] = 1; - start[0] = frame[nv]; - } - else if (fndims == iodesc_ndims) - { - start[0] += vdesc->record; - } - } - - /* Call the netCDF functions to write the data. */ - ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr); - - if (ierr) - { - for (int i = 0; i < fndims; i++) - fprintf(stderr, "vid %d dim %d start %ld count %ld \n", vid[nv], i, - start[i], count[i]); - return check_netcdf(file, ierr, __FILE__, __LINE__); - } - } /* next var */ - - /* Calculate the total size. */ - size_t tsize = 1; - for (int i = 0; i < fndims; i++) - tsize *= count[i]; - - /* Keep track of where we are in the buffer. */ - loffset += tsize; - - LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt, - tsize, loffset)); - } /* next regioncnt */ - } /* endif (rlen > 0) */ - } /* next rtask */ + if ((ierr = recv_and_write_data(file, vid, frame, iodesc, llen, num_regions, nvars, fndims, + tmp_start, tmp_count, iobuf))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } } @@ -692,7 +806,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu var_desc_t *vdesc; /* Information about the variable. */ int ndims; /* Number of dims in decomposition. */ int fndims; /* Number of dims for this var in file. */ - int mpierr; /* Return code from MPI functions. */ int ierr; /* Return code from netCDF functions. */ /* Check inputs. */ @@ -729,7 +842,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu size_t count[fndims]; size_t tmp_bufsize = 1; void *bufptr; - int tsize; int rrlen = 0; PIO_Offset *startlist[iodesc->maxregions]; PIO_Offset *countlist[iodesc->maxregions]; @@ -739,10 +851,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu the basetype. */ region = iodesc->firstregion; - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - /* ??? */ if (fndims > ndims) { @@ -771,7 +879,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu if (regioncnt == 0 || region == NULL) bufptr = iobuf; else - bufptr=(void *)((char *)iobuf + tsize * region->loffset); + bufptr=(void *)((char *)iobuf + iodesc->basetype_size * region->loffset); LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset)); @@ -932,17 +1040,12 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, size_t tmp_count[fndims * iodesc->maxregions]; size_t tmp_bufsize; void *bufptr; - int tsize; /* buffer is incremented by byte and loffset is in terms of the iodessc->basetype so we need to multiply by the size of the basetype. */ region = iodesc->firstregion; - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if (fndims > ndims) { if (vdesc->record < 0) @@ -1074,7 +1177,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, for (int regioncnt = 0; regioncnt < maxregions; regioncnt++) { /* Get pointer where data should go. */ - bufptr = (void *)((char *)iobuf + tsize * loffset); + bufptr = (void *)((char *)iobuf + iodesc->basetype_size * loffset); regionsize = 1; /* ??? */ @@ -1151,7 +1254,9 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) /* Find out the buffer usage. */ if ((ierr = ncmpi_inq_buffer_usage(file->fh, &usage))) - return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__); + /* allow the buffer to be undefined */ + if (ierr != NC_ENULLABUF) + return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__); /* If we are not forcing a flush, spread the usage to all IO * tasks. */ @@ -1223,6 +1328,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) vdesc = file->varlist + i; if (vdesc->iobuf) { + LOG((3,"freeing variable buffer in flush_output_buffer")); brel(vdesc->iobuf); vdesc->iobuf = NULL; } @@ -1298,6 +1404,7 @@ void free_cn_buffer_pool(iosystem_desc_t *ios) { #if !PIO_USE_MALLOC LOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool)); + /* Note: it is possible that CN_bpool has been freed and set to NULL by bpool_free() */ if (CN_bpool) { cn_buffer_report(ios, false); @@ -1312,9 +1419,9 @@ void free_cn_buffer_pool(iosystem_desc_t *ios) /** * Flush the buffer. * - * @param ncid identifies the netCDF file - * @param wmb May be NULL, in which case function returns. - * @param flushtodisk + * @param ncid identifies the netCDF file. + * @param wmb pointer to the wmulti_buffer structure. + * @param flushtodisk if true, then flush data to disk. * @returns 0 for success, error code otherwise. * @ingroup PIO_write_darray */ @@ -1333,14 +1440,14 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) LOG((1, "flush_buffer ncid = %d flushtodisk = %d", ncid, flushtodisk)); /* If there are any variables in this buffer... */ - if (wmb->validvars > 0) + if (wmb->num_arrays > 0) { /* Write any data in the buffer. */ - ret = PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, + ret = PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->num_arrays, wmb->arraylen, wmb->data, wmb->frame, wmb->fillvalue, flushtodisk); - wmb->validvars = 0; + wmb->num_arrays = 0; /* Release the list of variable IDs. */ brel(wmb->vid); @@ -1368,11 +1475,11 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) } /** - * Compute the maximum aggregate number of bytes. + * Compute the maximum aggregate number of bytes. This is called by + * subset_rearrange_create() and box_rearrange_create(). * - * @param ios the IO system structure - * @param iodesc a pointer to the defined iodescriptor for the - * buffer. If NULL, function returns immediately. + * @param ios pointer to the IO system structure. + * @param iodesc a pointer to decomposition description. * @returns 0 for success, error code otherwise. */ int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc) @@ -1388,19 +1495,27 @@ int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc) LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d", iodesc->maxiobuflen, iodesc->ndof)); + /* Determine the max bytes that can be held on IO task. */ if (ios->ioproc && iodesc->maxiobuflen > 0) maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen; + /* Determine the max bytes that can be held on computation task. */ if (ios->comp_rank >= 0 && iodesc->ndof > 0) maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof; + /* Take the min of the max IO and max comp bytes. */ maxbytes = min(maxbytesoniotask, maxbytesoncomputetask); LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d", maxbytesoniotask, maxbytesoncomputetask)); + /* Get the min value of this on all tasks. */ + LOG((3, "before allreaduce maxbytes = %d", maxbytes)); if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios->union_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + LOG((3, "after allreaduce maxbytes = %d", maxbytes)); + + /* Remember the result. */ iodesc->maxbytes = maxbytes; return PIO_NOERR; diff --git a/src/externals/pio2/src/clib/pio_file.c b/src/externals/pio2/src/clib/pio_file.c index f799e25f57cc..c0523997cfd4 100644 --- a/src/externals/pio2/src/clib/pio_file.c +++ b/src/externals/pio2/src/clib/pio_file.c @@ -106,8 +106,10 @@ int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename, return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Run this on all tasks if async is not in use, but only on - * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + * non-IO tasks if async is in use. (Because otherwise, in async + * mode, set_fill would be called twice by each IO task, since + * PIOc_createfile() will already be called on each IO task.) */ + if (!ios->async || !ios->ioproc) { /* Set the fill mode to NOFILL. */ if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL))) @@ -174,7 +176,7 @@ int PIOc_closefile(int ncid) /* Sync changes before closing on all tasks if async is not in * use, but only on non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) if (file->mode & PIO_WRITE) PIOc_sync(ncid); @@ -182,7 +184,7 @@ int PIOc_closefile(int ncid) * sends a msg to the pio_msg_handler running on the IO master and * waiting for a message. Then broadcast the ncid over the intercomm * to the IO tasks. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -264,7 +266,7 @@ int PIOc_deletefile(int iosysid, const char *filename) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* If async is in use, send message to IO master task. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -338,7 +340,7 @@ int PIOc_sync(int ncid) ios = file->iosystem; /* If async is in use, send message to IO master tasks. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -366,7 +368,9 @@ int PIOc_sync(int ncid) wmb = &file->buffer; while (wmb) { - if (wmb->validvars > 0) + /* If there are any data arrays waiting in the + * multibuffer, flush it. */ + if (wmb->num_arrays > 0) flush_buffer(ncid, wmb, true); twmb = wmb; wmb = wmb->next; diff --git a/src/externals/pio2/src/clib/pio_get_nc.c b/src/externals/pio2/src/clib/pio_get_nc.c index e8d542114309..2cd85d8f9476 100644 --- a/src/externals/pio2/src/clib/pio_get_nc.c +++ b/src/externals/pio2/src/clib/pio_get_nc.c @@ -619,7 +619,7 @@ int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, */ int PIOc_get_var_text(int ncid, int varid, char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_CHAR, buf); + return PIOc_get_var_tc(ncid, varid, NC_CHAR, buf); } /** @@ -635,7 +635,7 @@ int PIOc_get_var_text(int ncid, int varid, char *buf) */ int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UBYTE, buf); + return PIOc_get_var_tc(ncid, varid, NC_UBYTE, buf); } /** @@ -651,7 +651,7 @@ int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf) */ int PIOc_get_var_schar(int ncid, int varid, signed char *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_BYTE, buf); + return PIOc_get_var_tc(ncid, varid, NC_BYTE, buf); } /** @@ -667,7 +667,7 @@ int PIOc_get_var_schar(int ncid, int varid, signed char *buf) */ int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, buf); + return PIOc_get_var_tc(ncid, varid, NC_USHORT, buf); } /** @@ -683,7 +683,7 @@ int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf) */ int PIOc_get_var_short(int ncid, int varid, short *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_SHORT, buf); + return PIOc_get_var_tc(ncid, varid, NC_SHORT, buf); } /** @@ -699,7 +699,7 @@ int PIOc_get_var_short(int ncid, int varid, short *buf) */ int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT, buf); + return PIOc_get_var_tc(ncid, varid, NC_UINT, buf); } /** @@ -715,7 +715,7 @@ int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf) */ int PIOc_get_var_int(int ncid, int varid, int *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT, buf); + return PIOc_get_var_tc(ncid, varid, NC_INT, buf); } /** @@ -731,7 +731,7 @@ int PIOc_get_var_int(int ncid, int varid, int *buf) */ int PIOc_get_var_long (int ncid, int varid, long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, PIO_LONG_INTERNAL, buf); + return PIOc_get_var_tc(ncid, varid, PIO_LONG_INTERNAL, buf); } /** @@ -747,7 +747,7 @@ int PIOc_get_var_long (int ncid, int varid, long *buf) */ int PIOc_get_var_float(int ncid, int varid, float *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_FLOAT, buf); + return PIOc_get_var_tc(ncid, varid, NC_FLOAT, buf); } /** @@ -763,7 +763,7 @@ int PIOc_get_var_float(int ncid, int varid, float *buf) */ int PIOc_get_var_double(int ncid, int varid, double *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_DOUBLE, buf); + return PIOc_get_var_tc(ncid, varid, NC_DOUBLE, buf); } /** @@ -779,7 +779,7 @@ int PIOc_get_var_double(int ncid, int varid, double *buf) */ int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT64, buf); + return PIOc_get_var_tc(ncid, varid, NC_UINT64, buf); } /** @@ -795,7 +795,7 @@ int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf) */ int PIOc_get_var_longlong(int ncid, int varid, long long *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf); + return PIOc_get_var_tc(ncid, varid, NC_INT64, buf); } /** @@ -1044,7 +1044,7 @@ int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, */ int PIOc_get_var(int ncid, int varid, void *buf) { - return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, buf); + return PIOc_get_var_tc(ncid, varid, NC_NAT, buf); } /** diff --git a/src/externals/pio2/src/clib/pio_getput_int.c b/src/externals/pio2/src/clib/pio_getput_int.c index 32f1968186f8..187f74d058d6 100644 --- a/src/externals/pio2/src/clib/pio_getput_int.c +++ b/src/externals/pio2/src/clib/pio_getput_int.c @@ -52,7 +52,7 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype, /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the length (in bytes) of the type in file. */ if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len))) @@ -70,7 +70,7 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype, } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -259,7 +259,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type and length of the attribute. */ if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, &attlen))) @@ -284,7 +284,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void /* If async is in use, and this is not an IO task, bcast the * parameters and the attribute and type information we fetched. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -485,19 +485,19 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ - int ierr = PIO_NOERR; /* Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ - int ndims; /* The number of dimensions in the variable. */ - PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ + int ndims; /* The number of dimensions in the variable. */ + PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ + nc_type vartype; /* The type of the var we are reading from. */ char start_present = start ? true : false; char count_present = count ? true : false; char stride_present = stride ? true : false; - PIO_Offset *rstart = NULL, *rcount = NULL; - nc_type vartype; /* The type of the var we are reading from. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + int ierr; /* Return code. */ - LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d " - "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); + LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d xtype = %d start_present = %d " + "count_present = %d stride_present = %d", ncid, varid, xtype, start_present, + count_present, stride_present)); /* Find the info about this file. */ if ((ierr = pio_get_file(ncid, &file))) @@ -510,7 +510,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type of this var. */ if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype))) @@ -532,63 +532,20 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Get the number of dims for this var. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((3, "ndims = %d", ndims)); - PIO_Offset dimlen[ndims]; - - /* If no count array was passed, we need to know the dimlens - * so we can calculate how many data elements are in the - * buf. */ - if (!count) - { - int dimid[ndims]; - - /* Get the dimids for this var. */ - if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - - /* Get the length of each dimension. */ - for (int vd = 0; vd < ndims; vd++) - if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - } - - /* Figure out the real start, count, and stride arrays. (The - * user may have passed in NULLs.) */ - /* Allocate memory for these arrays, now that we know ndims. */ - if (!(rstart = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rcount = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - - PIO_Offset rstride[ndims]; - for (int vd = 0; vd < ndims; vd++) - { - rstart[vd] = start ? start[vd] : 0; - rcount[vd] = count ? count[vd] : dimlen[vd]; - rstride[vd] = stride ? stride[vd] : 1; - LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd, - rstart[vd], vd, rcount[vd], vd, rstride[vd])); - } + /* Only scalar vars can pass NULL for start/count. */ + pioassert(ndims == 0 || (start && count), "need start/count", __FILE__, __LINE__); - /* How many elements in buf? */ + /* How many elements in buf? (For scalars, ndims is 0 and + * num_elem will remain 1). */ for (int vd = 0; vd < ndims; vd++) - num_elem *= rcount[vd]; + num_elem *= count[vd]; LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem)); - - /* Free tmp resources. */ - if (start_present) - free(rstart); - else - start = rstart; - - if (count_present) - free(rcount); - else - count = rcount; } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -606,8 +563,12 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); @@ -619,9 +580,9 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d " - "stride_present = %d xtype = %d num_elem = %d", ncid, varid, - ndims, stride_present, xtype, num_elem)); + LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, + ndims, start_present, count_present, stride_present, xtype, num_elem)); } /* Handle MPI errors. */ @@ -646,7 +607,10 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - ncmpi_begin_indep_data(file->fh); + LOG((2, "pnetcdf calling ncmpi_get_vars_*() file->fh = %d varid = %d", file->fh, varid)); + /* Turn on independent access for pnetcdf file. */ + if ((ierr = ncmpi_begin_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Only the IO master does the IO, so we are not really * getting parallel IO here. */ @@ -678,8 +642,11 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off default: return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); } - }; - ncmpi_end_indep_data(file->fh); + } + + /* Turn off independent access for pnetcdf file. */ + if ((ierr = ncmpi_end_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } #endif /* _PNETCDF */ @@ -746,15 +713,6 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } } - if (!ios->async_interface || !ios->ioproc) - { - /* Free tmp start/count allocated to account for NULL start/counts */ - if (!start_present) - free(rstart); - if (!count_present) - free(rcount); - } - /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); @@ -772,7 +730,8 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /** - * Get one value of a variable of any type. + * Get one value of a variable of any type. This is an internal + * function. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -811,6 +770,74 @@ int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype return PIOc_get_vars_tc(ncid, varid, index, count, NULL, xtype, buf); } +/** + * Get a complete variable of any type. This is an internal function. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param index an array of start indicies (must have same number of + * entries as variable has dimensions). If NULL, indices of 0 will be + * used. + * @param xtype the netcdf type of the variable. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + PIO_Offset *startp = NULL; /* Pointer to start array. */ + PIO_Offset *countp = NULL; /* Pointer to count array. */ + int ndims; /* The number of dimensions in the variable. */ + PIO_Offset my_start[PIO_MAX_DIMS]; + PIO_Offset dimlen[PIO_MAX_DIMS]; + int ierr; /* Return code from function calls. */ + + LOG((1, "PIOc_get_var_tc ncid = %d varid = %d xtype = %d", ncid, varid, + xtype)); + + /* Find the info about this file. We need this for error handling. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Scalar vars (which have ndims == 0) should just pass NULLs for + * start/count. */ + if (ndims) + { + /* Find the dimension IDs. */ + int dimids[ndims]; + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Find the dimension lengths. */ + for (int d = 0; d < ndims; d++) + if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &dimlen[d]))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Set up start array. */ + for (int d = 0; d < ndims; d++) + { + my_start[d] = 0; + LOG((3, "my_start[%d] = %d dimlen[%d] = %d", d, my_start[d], d, + dimlen[d])); + } + + /* Set the start/count arrays. */ + startp = my_start; + countp = dimlen; + } + + return PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf); +} + /** * Internal PIO function which provides a type-neutral interface to * nc_put_vars. @@ -850,22 +877,22 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off const PIO_Offset *stride, nc_type xtype, const void *buf) { iosystem_desc_t *ios; /* Pointer to io system information. */ - file_desc_t *file; /* Pointer to file information. */ - int ndims; /* The number of dimensions in the variable. */ + file_desc_t *file; /* Pointer to file information. */ + int ndims; /* The number of dimensions in the variable. */ PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ - char start_present = start ? true : false; /* Is start non-NULL? */ - char count_present = count ? true : false; /* Is count non-NULL? */ - char stride_present = stride ? true : false; /* Is stride non-NULL? */ - PIO_Offset *rstart, *rcount, *rstride; + char start_present = start ? true : false; /* Is start non-NULL? */ + char count_present = count ? true : false; /* Is count non-NULL? */ + char stride_present = stride ? true : false; /* Is stride non-NULL? */ var_desc_t *vdesc; int *request; nc_type vartype; /* The type of the var we are reading from. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr; /* Return code from function calls. */ - LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start = %d count = %d " - "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); + LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d", ncid, varid, + start_present, count_present, stride_present, xtype)); /* Get file info. */ if ((ierr = pio_get_file(ncid, &file))) @@ -878,7 +905,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Get the type of this var. */ if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype))) @@ -903,69 +930,15 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "ndims = %d typelen = %d", ndims, typelen)); - PIO_Offset dimlen[ndims]; - - /* If no count array was passed, we need to know the dimlens - * so we can calculate how many data elements are in the - * buf. */ - if (!count) - { - int dimid[ndims]; - - /* Get the dimids for this var. */ - if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - - /* Get the length of each dimension. */ + /* How many elements of data? If no count array was passed, + * this is a scalar. */ + if (count) for (int vd = 0; vd < ndims; vd++) - { - if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - LOG((3, "dimlen[%d] = %d", vd, dimlen[vd])); - } - } - - /* Allocate memory for these arrays, now that we know ndims. */ - if (!(rstart = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rcount = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(rstride = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - - /* Figure out the real start, count, and stride arrays. (The - * user may have passed in NULLs.) */ - for (int vd = 0; vd < ndims; vd++) - { - rstart[vd] = start ? start[vd] : 0; - rcount[vd] = count ? count[vd] : dimlen[vd]; - rstride[vd] = stride ? stride[vd] : 1; - LOG((3, "rstart[%d] = %d rcount[%d] = %d rstride[%d] = %d", vd, - rstart[vd], vd, rcount[vd], vd, rstride[vd])); - } - - /* How many elements in buf? */ - for (int vd = 0; vd < ndims; vd++) - num_elem *= rcount[vd]; - LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); - - /* Free tmp resources. */ - if (start_present) - free(rstart); - else - start = rstart; - - if (count_present) - free(rcount); - else - count = rcount; - - /* Only PNETCDF requires a non-NULL stride, realocate it later if needed */ - free(rstride); + num_elem *= count[vd]; } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -983,8 +956,12 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); @@ -1028,69 +1005,121 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - PIO_Offset *fake_stride; - - if (!stride_present) + /* Scalars have to be handled differently. */ + if (ndims == 0) { - LOG((2, "stride not present")); - if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - for (int d = 0; d < ndims; d++) - fake_stride[d] = 1; + /* This is a scalar var. */ + LOG((2, "pnetcdf writing scalar with ncmpi_put_vars_*() file->fh = %d varid = %d", + file->fh, varid)); + pioassert(!start && !count && !stride, "expected NULLs", __FILE__, __LINE__); + + /* Turn on independent access for pnetcdf file. */ + if ((ierr = ncmpi_begin_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Only the IO master does the IO, so we are not really + * getting parallel IO here. */ + if (ios->iomaster == MPI_ROOT) + { + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_put_vars_schar(file->fh, varid, start, count, stride, buf); + break; + case NC_CHAR: + ierr = ncmpi_put_vars_text(file->fh, varid, start, count, stride, buf); + break; + case NC_SHORT: + ierr = ncmpi_put_vars_short(file->fh, varid, start, count, stride, buf); + break; + case NC_INT: + ierr = ncmpi_put_vars_int(file->fh, varid, start, count, stride, buf); + break; + case PIO_LONG_INTERNAL: + ierr = ncmpi_put_vars_long(file->fh, varid, start, count, stride, buf); + break; + case NC_FLOAT: + ierr = ncmpi_put_vars_float(file->fh, varid, start, count, stride, buf); + break; + case NC_DOUBLE: + ierr = ncmpi_put_vars_double(file->fh, varid, start, count, stride, buf); + break; + default: + return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); + } + } + + /* Turn off independent access for pnetcdf file. */ + if ((ierr = ncmpi_end_indep_data(file->fh))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } else - fake_stride = (PIO_Offset *)stride; - - LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); - vdesc = file->varlist + varid; - if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) - if (!(vdesc->request = realloc(vdesc->request, - sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - request = vdesc->request + vdesc->nreqs; - LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); - - /* Only the IO master actually does the call. */ - if (ios->iomaster == MPI_ROOT) { - switch(xtype) + /* This is not a scalar var. */ + PIO_Offset *fake_stride; + + if (!stride_present) { - case NC_BYTE: - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_CHAR: - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_SHORT: - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_INT: - ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request); - break; - case PIO_LONG_INTERNAL: - ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_FLOAT: - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request); - break; - case NC_DOUBLE: - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request); - break; - default: - return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + LOG((2, "stride not present")); + if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + for (int d = 0; d < ndims; d++) + fake_stride[d] = 1; } - LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr)); - } - else - *request = PIO_REQ_NULL; + else + fake_stride = (PIO_Offset *)stride; + + LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); + vdesc = &file->varlist[varid]; + if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) + if (!(vdesc->request = realloc(vdesc->request, + sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + request = vdesc->request + vdesc->nreqs; + LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); + + /* Only the IO master actually does the call. */ + if (ios->iomaster == MPI_ROOT) + { + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_CHAR: + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_SHORT: + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_INT: + ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request); + break; + case PIO_LONG_INTERNAL: + ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_FLOAT: + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request); + break; + case NC_DOUBLE: + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request); + break; + default: + return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + } + LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr)); + } + else + *request = PIO_REQ_NULL; - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - LOG((2, "PIOc_put_vars_tc flushed output buffer")); + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + LOG((2, "PIOc_put_vars_tc flushed output buffer")); - /* Free malloced resources. */ - if (!stride_present) - free(fake_stride); + /* Free malloced resources. */ + if (!stride_present) + free(fake_stride); + } /* endif ndims == 0 */ } #endif /* _PNETCDF */ @@ -1161,15 +1190,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } } - if (!ios->async_interface || !ios->ioproc) - { - /* Free tmp start/count allocated to account for NULL start/counts */ - if (!start_present) - free(rstart); - if (!count_present) - free(rcount); - } - /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); @@ -1232,3 +1252,78 @@ int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype return PIOc_put_vars_tc(ncid, varid, index, count, NULL, xtype, op); } + +/** + * Internal PIO function which provides a type-neutral interface to + * nc_put_var calls. + * + * Users should not call this function directly. Instead, call one of + * the derived functions, depending on the type of data you are + * writing: PIOc_put_var_text(), PIOc_put_var_uchar(), + * PIOc_put_var_schar(), PIOc_put_var_ushort(), + * PIOc_put_var_short(), PIOc_put_var_uint(), PIOc_put_var_int(), + * PIOc_put_var_long(), PIOc_put_var_float(), + * PIOc_put_var_longlong(), PIOc_put_var_double(), + * PIOc_put_var_ulonglong(). + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from this type to the type of the + * variable being written to. + * @param op pointer to the data to be written. + * + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + PIO_Offset *startp = NULL; /* Pointer to start array. */ + PIO_Offset *countp = NULL; /* Pointer to count array. */ + PIO_Offset start[PIO_MAX_DIMS]; + PIO_Offset count[PIO_MAX_DIMS]; + int ndims; /* The number of dimensions in the variable. */ + int ierr; /* Return code from function calls. */ + + LOG((1, "PIOc_put_var_tc ncid = %d varid = %d xtype = %d", ncid, + varid, xtype)); + + /* Find the info about this file. We need this for error handling. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Scalar vars (which have ndims == 0) should just pass NULLs for + * start/count. */ + if (ndims) + { + int dimid[ndims]; + + /* Set up start array. */ + for (int d = 0; d < ndims; d++) + start[d] = 0; + + /* Get the dimids for this var. */ + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Count array are the dimlens. */ + for (int d = 0; d < ndims; d++) + if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &count[d]))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); + + /* Set the array pointers. */ + startp = start; + countp = count; + } + + return PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op); +} diff --git a/src/externals/pio2/src/clib/pio_internal.h b/src/externals/pio2/src/clib/pio_internal.h index e1c891c761e0..9f1a4a18d5db 100644 --- a/src/externals/pio2/src/clib/pio_internal.h +++ b/src/externals/pio2/src/clib/pio_internal.h @@ -12,6 +12,15 @@ #include +/* These are the sizes of types in netCDF files. Do not replace these + * constants with sizeof() calls for C types. They are not the + * same. Even on a system where sizeof(short) is 4, the size of a + * short in a netCDF file is 2 bytes. */ +#define NETCDF_CHAR_SIZE 1 +#define NETCDF_SHORT_SIZE 2 +#define NETCDF_INT_FLOAT_SIZE 4 +#define NETCDF_DOUBLE_INT64_SIZE 8 + /* It seems that some versions of openmpi fail to define * MPI_OFFSET. */ #ifdef OMPI_OFFSET_DATATYPE @@ -123,8 +132,8 @@ extern "C" { int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status, const char *fname, int line); - /* Find the MPI type that matches a PIO type. */ - int find_mpi_type(int pio_type, MPI_Datatype *mpi_type); + /* Given PIO type, find MPI type and type size. */ + int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size); /* Check whether an IO type is valid for this build. */ int iotype_is_valid(int iotype); @@ -135,30 +144,59 @@ extern "C" { /* Assert that an expression is true. */ void pioassert(bool exp, const char *msg, const char *fname, int line); - int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs, - int myiorank, PIO_Offset *start, PIO_Offset *kount); + /* Compute start and count values for each io task for a decomposition. */ + int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs, + int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks); + + /* Completes the mapping for the box rearranger. */ + int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, const int *dest_ioproc, + const PIO_Offset *dest_ioindex); + + /* Create the MPI communicators needed by the subset rearranger. */ + int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc); /* Check return from MPI function and print error message. */ void CheckMPIReturn(int ierr, const char *file, int line); /* Like MPI_Alltoallw(), but with flow control. */ - int pio_swapm(void *sndbuf, int *sndlths, int *sdispls, MPI_Datatype *stypes, - void *rcvbuf, int *rcvlths, int *rdispls, MPI_Datatype *rtypes, - MPI_Comm comm, bool handshake, bool isend, int max_requests); + int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes, + void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes, + MPI_Comm comm, rearr_comm_fc_opt_t *fc); long long lgcd_array(int nain, long long* ain); void PIO_Offset_size(MPI_Datatype *dtype, int *tsize); PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in); - /* Initialize the rearranger options. */ - void init_rearr_opts(iosystem_desc_t *iosys); + /* Convert an index into dimension values. */ + void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, PIO_Offset *dim_list); + + /* Convert a global coordinate value into a local array index. */ + PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count); + + /* Determine whether fill values are needed. */ + int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, + const PIO_Offset *compmap); + + /* Set start and count so that they describe the first region in map.*/ + PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map, + PIO_Offset *start, PIO_Offset *count); + + /* Calculate start and count regions for the subset rearranger. */ + int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, + int *maxregions, io_region *firstregion); + + /* Expand a region along dimension dim, by incrementing count[i] as + * much as possible, consistent with the map. */ + void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, + int region_size, int region_stride, const int *max_size, + PIO_Offset *count); /* Compare sets of rearranger options. */ bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts); - /* Reset rearranger opts in iosystem to valid values. */ - void check_and_reset_rearr_opts(iosystem_desc_t *ios); + /* Check and reset, if needed, rearranger opts to default values. */ + int check_and_reset_rearr_opts(rearr_opt_t *rearr_opt); /* Compare rearranger flow control options. */ bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt, @@ -178,15 +216,23 @@ extern "C" { int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf); /* Move data from compute tasks to IO tasks. */ - int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, int nvars); + int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, + int nvars); /* Allocate and initialize storage for decomposition information. */ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc); void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc); + /* Flush contents of multi-buffer to disk. */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); + + /* Compute the size that the IO tasks will need to hold the data. */ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); - io_region *alloc_region(int ndims); + + /* Allocation memory for a data region. */ + int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **region); + + /* Delete an entry from the lost of open IO systems. */ int pio_delete_iosystem_from_list(int piosysid); /* Find greatest commond divisor. */ @@ -211,7 +257,7 @@ extern "C" { /* Create the derived MPI datatypes used for comp2io and io2comp * transfers. */ - int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, const PIO_Offset *mindex, + int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, const PIO_Offset *mindex, const int *mcount, int *mfrom, MPI_Datatype *mtype); int compare_offsets(const void *a, const void *b) ; @@ -230,8 +276,9 @@ extern "C" { int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc); - /* Announce a memory error with bget memory, and die. */ - void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line); + /* Compute an element of start/count arrays. */ + void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, + PIO_Offset *count); /* Check the return code from an MPI function call. */ int check_mpi(file_desc_t *file, int mpierr, const char *filename, int line); @@ -249,10 +296,8 @@ extern "C" { const int *frame); /* Write aggregated arrays to file using serial I/O (netCDF-3/netCDF-4 serial) */ - int pio_write_darray_multi_nc_serial(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims, - MPI_Datatype basetype, int maxregions, io_region *firstregion, - PIO_Offset llen, int num_aiotasks, void *iobuf, - const int *frame); + int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid, + io_desc_t *iodesc, int fill, const int *frame); int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); @@ -269,13 +314,16 @@ extern "C" { const PIO_Offset *stride, nc_type xtype, void *buf); int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, void *buf); + int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf); + /* Generalized put functions. */ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, nc_type xtype, const void *buf); int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, const void *op); - + int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op); + /* An internal replacement for a function pnetcdf does not * have. */ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, @@ -289,9 +337,9 @@ extern "C" { void pio_finalize_logging(void ); /* Write a netCDF decomp file. */ - int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims, int *global_dimlen, - int num_tasks, int *task_maplen, int *map, const char *title, - const char *history, int fortran_order); + int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims, + int *global_dimlen, int num_tasks, int *task_maplen, int *map, + const char *title, const char *history, int fortran_order); /* Read a netCDF decomp file. */ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen, diff --git a/src/externals/pio2/src/clib/pio_lists.c b/src/externals/pio2/src/clib/pio_lists.c index 72e774715bc5..df79302cfd19 100644 --- a/src/externals/pio2/src/clib/pio_lists.c +++ b/src/externals/pio2/src/clib/pio_lists.c @@ -115,8 +115,14 @@ int pio_delete_file_from_list(int ncid) if (current_file == cfile) current_file = pfile; + /* Free any fill values that were allocated. */ + for (int v = 0; v < PIO_MAX_VARS; v++) + if (cfile->varlist[v].fillvalue) + free(cfile->varlist[v].fillvalue); + /* Free the memory used for this file. */ free(cfile); + return PIO_NOERR; } pfile = cfile; diff --git a/src/externals/pio2/src/clib/pio_msg.c b/src/externals/pio2/src/clib/pio_msg.c index 75d2a8bfea4f..b8ca0e9917b0 100644 --- a/src/externals/pio2/src/clib/pio_msg.c +++ b/src/externals/pio2/src/clib/pio_msg.c @@ -675,16 +675,18 @@ int put_vars_handler(iosystem_desc_t *ios) { int ncid; int varid; - PIO_Offset typelen; /** Length (in bytes) of this type. */ - nc_type xtype; /** Type of the data being written. */ - char stride_present; /** Zero if user passed a NULL stride. */ + PIO_Offset typelen; /* Length (in bytes) of this type. */ + nc_type xtype; /* Type of the data being written. */ + char start_present; /* Zero if user passed a NULL start. */ + char count_present; /* Zero if user passed a NULL count. */ + char stride_present; /* Zero if user passed a NULL stride. */ PIO_Offset *startp = NULL; PIO_Offset *countp = NULL; PIO_Offset *stridep = NULL; - int ndims; /** Number of dimensions. */ - void *buf; /** Buffer for data storage. */ - PIO_Offset num_elem; /** Number of data elements in the buffer. */ - int mpierr; /** Error code from MPI function calls. */ + int ndims; /* Number of dimensions. */ + void *buf; /* Buffer for data storage. */ + PIO_Offset num_elem; /* Number of data elements in the buffer. */ + int mpierr; /* Error code from MPI function calls. */ LOG((1, "put_vars_handler")); assert(ios); @@ -701,11 +703,17 @@ int put_vars_handler(iosystem_desc_t *ios) /* Now we know how big to make these arrays. */ PIO_Offset start[ndims], count[ndims], stride[ndims]; - if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (start_present) + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); - if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (count_present) + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if (stride_present) @@ -718,16 +726,9 @@ int put_vars_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d " - "stride_present = %d xtype = %d num_elem = %d typelen = %d", - ncid, varid, ndims, stride_present, xtype, num_elem, typelen)); - - for (int d = 0; d < ndims; d++) - { - LOG((2, "start[%d] = %d", d, start[d])); - LOG((2, "count[%d] = %d", d, count[d])); - if (stride_present) - LOG((2, "stride[%d] = %d", d, stride[d])); - } + "start_present = %d count_present = %d stride_present = %d xtype = %d " + "num_elem = %d typelen = %d", ncid, varid, ndims, start_present, count_present, + stride_present, xtype, num_elem, typelen)); /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) @@ -735,17 +736,13 @@ int put_vars_handler(iosystem_desc_t *ios) /* Get the data. */ if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm))) - { - free(buf); return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - } - - /* for (int e = 0; e < num_elem; e++) */ - /* LOG((2, "element %d = %d", e, ((int *)buf)[e])); */ /* Set the non-NULL pointers. */ - startp = start; - countp = count; + if (start_present) + startp = start; + if (count_present) + countp = count; if (stride_present) stridep = stride; @@ -819,6 +816,11 @@ int get_vars_handler(iosystem_desc_t *ios) int mpierr; PIO_Offset typelen; /** Length (in bytes) of this type. */ nc_type xtype; /** Type of the data being written. */ + PIO_Offset *start; + PIO_Offset *count; + PIO_Offset *stride; + char start_present; + char count_present; char stride_present; PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; int ndims; /** Number of dimensions. */ @@ -836,20 +838,33 @@ int get_vars_handler(iosystem_desc_t *ios) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - - /* Now we know how big to make these arrays. */ - PIO_Offset start[ndims], count[ndims], stride[ndims]; - - if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); - if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + if (start_present) + { + if (!(start = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if (count_present) + { + if (!(count = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if (stride_present) + { + if (!(stride = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm))) @@ -860,21 +875,17 @@ int get_vars_handler(iosystem_desc_t *ios) "stride_present = %d xtype = %d num_elem = %d typelen = %d", ncid, varid, ndims, stride_present, xtype, num_elem, typelen)); - for (int d = 0; d < ndims; d++) - { - LOG((2, "start[%d] = %d", d, start[d])); - LOG((2, "count[%d] = %d", d, count[d])); - if (stride_present) - LOG((2, "stride[%d] = %d", d, stride[d])); - } - /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Set the non-NULL pointers. */ - startp = start; - countp = count; + if (start_present) + startp = start; + + if (count_present) + countp = count; + if (stride_present) stridep = stride; @@ -928,7 +939,15 @@ int get_vars_handler(iosystem_desc_t *ios) #endif /* _NETCDF4 */ } + /* Free resourses. */ free(buf); + if (start_present) + free(start); + if (count_present) + free(count); + if (stride_present) + free(stride); + LOG((1, "get_vars_handler succeeded!")); return PIO_NOERR; } @@ -1889,7 +1908,7 @@ int delete_file_handler(iosystem_desc_t *ios) int initdecomp_dof_handler(iosystem_desc_t *ios) { int iosysid; - int basetype; + int pio_type; int ndims; int maplen; int ioid; @@ -1910,7 +1929,7 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&basetype, 1, MPI_INT, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1951,9 +1970,9 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(iocount, ndims, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((2, "initdecomp_dof_handler iosysid = %d basetype = %d ndims = %d maplen = %d " + LOG((2, "initdecomp_dof_handler iosysid = %d pio_type = %d ndims = %d maplen = %d " "rearranger_present = %d iostart_present = %d iocount_present = %d ", - iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present)); + iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); if (rearranger_present) rearrangerp = &rearranger; @@ -1963,7 +1982,7 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) iocountp = iocount; /* Call the function. */ - ret = PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, &ioid, rearrangerp, + ret = PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp, iostartp, iocountp); LOG((1, "PIOc_InitDecomp returned %d", ret)); diff --git a/src/externals/pio2/src/clib/pio_nc.c b/src/externals/pio2/src/clib/pio_nc.c index 8b7fb81ba646..00932bdfd64e 100644 --- a/src/externals/pio2/src/clib/pio_nc.c +++ b/src/externals/pio2/src/clib/pio_nc.c @@ -49,7 +49,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -89,9 +89,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - LOG((2, "PIOc_inq calling ncmpi_inq unlimdimidp = %d", unlimdimidp)); ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp); - LOG((2, "PIOc_inq called ncmpi_inq")); if (unlimdimidp) LOG((2, "PIOc_inq returned from ncmpi_inq unlimdimid = %d", *unlimdimidp)); } @@ -205,7 +203,7 @@ int PIOc_inq_natts(int ncid, int *ngattsp) */ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) { - LOG((1, "PIOc_inq_unlimdim ncid = %d unlimdimidp = %d", ncid, unlimdimidp)); + LOG((1, "PIOc_inq_unlimdim ncid = %d", ncid)); return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp); } @@ -234,7 +232,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -323,7 +321,7 @@ int PIOc_inq_format(int ncid, int *formatp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -402,7 +400,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -543,7 +541,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) LOG((1, "PIOc_inq_dimid ncid = %d name = %s", ncid, name)); /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -629,10 +627,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); ios = file->iosystem; - LOG((2, "got file and iosystem")); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -689,8 +686,6 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) { ierr = nc_inq_varndims(file->fh, varid, &ndims); - LOG((2, "file->fh = %d varid = %d xtypep = %d ndimsp = %d dimidsp = %d nattsp = %d", - file->fh, varid, xtypep, ndimsp, dimidsp, nattsp)); if (!ierr) { char my_name[NC_MAX_NAME + 1]; @@ -871,7 +866,7 @@ int PIOc_inq_varid(int ncid, const char *name, int *varidp) LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -957,11 +952,10 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, if (!name || strlen(name) > NC_MAX_NAME) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d", - ncid, varid, xtypep, lenp)); + LOG((1, "PIOc_inq_att ncid = %d varid = %d", ncid, varid)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1086,7 +1080,7 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1180,7 +1174,7 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1269,7 +1263,7 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1354,7 +1348,7 @@ int PIOc_rename_var(int ncid, int varid, const char *name) LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1443,7 +1437,7 @@ int PIOc_rename_att(int ncid, int varid, const char *name, ncid, varid, name, newname)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1531,7 +1525,7 @@ int PIOc_del_att(int ncid, int varid, const char *name) LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1601,8 +1595,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep) int ierr; /* Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ - LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode, - old_modep)); + LOG((1, "PIOc_set_fill ncid = %d fillmode = %d", ncid, fillmode)); /* Find the info about this file. */ if ((ierr = pio_get_file(ncid, &file))) @@ -1610,7 +1603,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1739,7 +1732,7 @@ int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp) LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1831,7 +1824,7 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, xtype, ndims)); /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -1948,7 +1941,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the size of this vars * type. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype))) return check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1958,7 +1951,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value LOG((2, "PIOc_def_var_fill type_size = %d", type_size)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -2042,7 +2035,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param fill_mode a pointer to int that will get the fill + * @param no_fill a pointer to int that will get the fill * mode. Ignored if NULL (except with pnetcdf, which seg-faults with * NULL.) * @param fill_valuep pointer to space that gets the fill value for @@ -2050,7 +2043,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value * @return PIO_NOERR for success, error code otherwise. * @ingroup PIO_inq_var_fill */ -int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) +int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) { iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; /* Pointer to file information. */ @@ -2070,7 +2063,7 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the size of this vars * type. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype))) return check_netcdf(file, ierr, __FILE__, __LINE__); @@ -2080,12 +2073,12 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { int msg = PIO_MSG_INQ_VAR_FILL; - char fill_mode_present = fill_mode ? true : false; + char no_fill_present = no_fill ? true : false; char fill_value_present = fill_valuep ? true : false; LOG((2, "sending msg type_size = %d", type_size)); @@ -2099,11 +2092,11 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) if (!mpierr) mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&no_fill_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld fill_mode_present = %d fill_value_present = %d", - ncid, varid, type_size, fill_mode_present, fill_value_present)); + LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld no_fill_present = %d fill_value_present = %d", + ncid, varid, type_size, no_fill_present, fill_value_present)); } /* Handle MPI errors. */ @@ -2122,22 +2115,22 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d fill_mode = %d", - file->iotype, file->fh, varid, fill_mode)); + LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d", + file->iotype, file->fh, varid)); if (file->iotype == PIO_IOTYPE_PNETCDF) { #ifdef _PNETCDF - ierr = ncmpi_inq_var_fill(file->fh, varid, fill_mode, fill_valuep); + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _PNETCDF */ } - else if (file->iotype == PIO_IOTYPE_NETCDF) + else if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) { /* Get the file-level fill mode. */ - if (fill_mode) + if (no_fill) { - ierr = nc_set_fill(file->fh, NC_NOFILL, fill_mode); + ierr = nc_set_fill(file->fh, NC_NOFILL, no_fill); if (!ierr) - ierr = nc_set_fill(file->fh, *fill_mode, NULL); + ierr = nc_set_fill(file->fh, *no_fill, NULL); } if (!ierr && fill_valuep) @@ -2183,7 +2176,7 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) #ifdef _NETCDF4 /* The inq_var_fill is not supported in classic-only builds. */ if (file->do_io) - ierr = nc_inq_var_fill(file->fh, varid, fill_mode, fill_valuep); + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _NETCDF */ } LOG((2, "after call to inq_var_fill, ierr = %d", ierr)); @@ -2196,8 +2189,8 @@ int PIOc_inq_var_fill(int ncid, int varid, int *fill_mode, void *fill_valuep) return check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ - if (fill_mode) - if ((mpierr = MPI_Bcast(fill_mode, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if (no_fill) + if ((mpierr = MPI_Bcast(no_fill, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr, __FILE__, __LINE__); if (fill_valuep) if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, ios->ioroot, ios->my_comm))) diff --git a/src/externals/pio2/src/clib/pio_nc4.c b/src/externals/pio2/src/clib/pio_nc4.c index b68bba41050d..0b1cdd0e6b7e 100644 --- a/src/externals/pio2/src/clib/pio_nc4.c +++ b/src/externals/pio2/src/clib/pio_nc4.c @@ -48,7 +48,7 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -137,7 +137,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -251,13 +251,13 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *ch /* Run this on all tasks if async is not in use, but only on * non-IO tasks if async is in use. Get the number of * dimensions. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); LOG((2, "PIOc_def_var_chunking first ndims = %d", ndims)); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -370,7 +370,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) + if (!ios->async || !ios->ioproc) { /* Find the number of dimensions of this variable. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) @@ -379,7 +379,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks } /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -495,7 +495,7 @@ int PIOc_def_var_endian(int ncid, int varid, int endian) return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -572,7 +572,7 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp) return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -663,7 +663,7 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset ne return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -761,7 +761,7 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -879,7 +879,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -966,7 +966,7 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { diff --git a/src/externals/pio2/src/clib/pio_put_nc.c b/src/externals/pio2/src/clib/pio_put_nc.c index 74279c9871e3..7c74144da3b4 100644 --- a/src/externals/pio2/src/clib/pio_put_nc.c +++ b/src/externals/pio2/src/clib/pio_put_nc.c @@ -860,7 +860,7 @@ int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, */ int PIOc_put_var_text(int ncid, int varid, const char *op) { - return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_CHAR, op); } /** @@ -882,7 +882,7 @@ int PIOc_put_var_text(int ncid, int varid, const char *op) */ int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) { - return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UBYTE, op); } /** @@ -904,7 +904,7 @@ int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) */ int PIOc_put_var_schar(int ncid, int varid, const signed char *op) { - return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_BYTE, op); } /** @@ -926,7 +926,7 @@ int PIOc_put_var_schar(int ncid, int varid, const signed char *op) */ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) { - return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); + return PIOc_put_var_tc(ncid, varid, NC_USHORT, op); } /** @@ -948,7 +948,7 @@ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) */ int PIOc_put_var_short(int ncid, int varid, const short *op) { - return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_SHORT, op); } /** @@ -970,7 +970,7 @@ int PIOc_put_var_short(int ncid, int varid, const short *op) */ int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) { - return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UINT, op); } /** @@ -992,7 +992,7 @@ int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) */ int PIOc_put_var_int(int ncid, int varid, const int *op) { - return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_INT, op); } /** @@ -1014,7 +1014,7 @@ int PIOc_put_var_int(int ncid, int varid, const int *op) */ int PIOc_put_var_long(int ncid, int varid, const long *op) { - return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_LONG_INTERNAL, op); } /** @@ -1036,7 +1036,7 @@ int PIOc_put_var_long(int ncid, int varid, const long *op) */ int PIOc_put_var_float(int ncid, int varid, const float *op) { - return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_FLOAT, op); } /** @@ -1058,7 +1058,7 @@ int PIOc_put_var_float(int ncid, int varid, const float *op) */ int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op) { - return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_UINT64, op); } /** @@ -1080,7 +1080,7 @@ int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op) */ int PIOc_put_var_longlong(int ncid, int varid, const long long *op) { - return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_INT64, op); } /** @@ -1102,7 +1102,7 @@ int PIOc_put_var_longlong(int ncid, int varid, const long long *op) */ int PIOc_put_var_double(int ncid, int varid, const double *op) { - return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_var_tc(ncid, varid, PIO_DOUBLE, op); } /** @@ -1118,7 +1118,7 @@ int PIOc_put_var_double(int ncid, int varid, const double *op) */ int PIOc_put_var(int ncid, int varid, const void *op) { - return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_NAT, op); + return PIOc_put_var_tc(ncid, varid, NC_NAT, op); } /** diff --git a/src/externals/pio2/src/clib/pio_rearrange.c b/src/externals/pio2/src/clib/pio_rearrange.c index 458e7e3844ff..74b27d45b06e 100644 --- a/src/externals/pio2/src/clib/pio_rearrange.c +++ b/src/externals/pio2/src/clib/pio_rearrange.c @@ -8,50 +8,40 @@ #include /** - * Internal library util function to initialize rearranger options. + * Convert a 1-D index into a coordinate value in an arbitrary + * dimension space. E.g., for index 4 into a array defined as a[3][2], + * will return 2,0. * - * @param iosys pointer to iosystem descriptor - */ -void init_rearr_opts(iosystem_desc_t *iosys) -{ - /* The old default for max pending requests was 64 - we no longer use it*/ - - /* Disable handshake /isend and set max_pend_req = 0 to turn of throttling */ - const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 }; - - assert(iosys); - - /* Default to coll - i.e., no flow control */ - iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL; - iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; - iosys->rearr_opts.comm_fc_opts_comp2io = def_coll_comm_fc_opts; - iosys->rearr_opts.comm_fc_opts_io2comp = def_coll_comm_fc_opts; -} - -/** - * Convert an index into a list of dimensions. E.g., for index 4 into a - * array defined as a[3][2], will return 1 1. + * Sometimes (from box_rearranger_create()) this function is called + * with -1 for idx. Not clear if this makes sense. * - * @param ndims number of dimensions - * @param gdims - * @param idx - * @param dim_list - * @returns 0 on success, error code otherwise. + * @param ndims number of dimensions. + * @param gdimlen array of length ndims with the dimension sizes. + * @param idx the index to convert. This is the index into a 1-D array + * of data. + * @param dim_list array of length ndims that will get the dimensions + * corresponding to this index. */ -void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, +void idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx, PIO_Offset *dim_list) { - int i, curr_idx, next_idx; - curr_idx = idx; + /* Check inputs. */ + pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input", + __FILE__, __LINE__); + LOG((2, "idx_to_dim_list ndims = %d idx = %d", ndims, idx)); /* Easiest to start from the right and move left. */ - for (i = ndims - 1; i >= 0; --i) + for (int i = ndims - 1; i >= 0; --i) { + int next_idx; + /* This way of doing div/mod is slightly faster than using "/" * and "%". */ - next_idx = curr_idx / gdims[i]; - dim_list[i] = curr_idx - (next_idx * gdims[i]); - curr_idx = next_idx; + next_idx = idx / gdimlen[i]; + dim_list[i] = idx - (next_idx * gdimlen[i]); + LOG((3, "next_idx = %d idx = %d gdimlen[%d] = %d dim_list[%d] = %d", + next_idx, idx, i, gdimlen[i], i, dim_list[i])); + idx = next_idx; } } @@ -64,30 +54,34 @@ void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, * outermost dimension, until the region has been expanded as much as * possible along all dimensions. * - * @param dim - * @param gdims array of dimension ids - * @param maplen the length of the map - * @param map - * @param region_size - * @param region_stride - * @param max_size array of maximum sizes - * @param count array of counts - * @returns 0 on success, error code otherwise. + * Precondition: maplen >= region_size (thus loop runs at least + * once). + * + * @param dim the dimension number to start with. + * @param gdimlen array of global dimension lengths. + * @param maplen the length of the map. + * @param map array (length maplen) with the the 1-based compmap. + * @param region_size ??? + * @param region_stride amount incremented along dimension. + * @param max_size array of size dim + 1 that contains the maximum + * sizes along that dimension. + * @param count array of size dim + 1 that gets the new counts. */ -void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, +void expand_region(int dim, const int *gdimlen, int maplen, const PIO_Offset *map, int region_size, int region_stride, const int *max_size, PIO_Offset *count) { - int i, j, test_idx, expansion_done; - /* Precondition: maplen >= region_size (thus loop runs at least - * once). */ - /* Flag used to signal that we can no longer expand the region along dimension dim. */ - expansion_done = 0; + int expansion_done = 0; + + /* Check inputs. */ + pioassert(dim >= 0 && gdimlen && maplen >= 0 && map && region_size >= 0 && + maplen >= region_size && region_stride >= 0 && max_size && count, + "invalid input", __FILE__, __LINE__); /* Expand no greater than max_size along this dimension. */ - for (i = 1; i <= max_size[dim]; ++i) + for (int i = 1; i <= max_size[dim]; ++i) { /* Count so far is at least i. */ count[dim] = i; @@ -97,8 +91,10 @@ void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, Assuming monotonicity in the map, we could skip this for the innermost dimension, but it's necessary past that because the region does not necessarily comprise contiguous values. */ - for (j = 0; j < region_size; ++j) + for (int j = 0; j < region_size; j++) { + int test_idx; /* Index we are testing. */ + test_idx = j + i * region_size; /* If we have exhausted the map, or the map no longer matches, @@ -116,65 +112,91 @@ void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map, /* Move on to next outermost dimension if there are more left, * else return. */ if (dim > 0) - expand_region(dim-1, gdims, maplen, map, region_size * count[dim], - region_stride * gdims[dim], max_size, count); + expand_region(dim - 1, gdimlen, maplen, map, region_size * count[dim], + region_stride * gdimlen[dim], max_size, count); } /** - * Set start and count so that they describe the first region in map. + * Set start and count so that they describe the first region in + * map. * - * @param ndims the number of dimensions - * @param gdims pointer to an array of dimension ids - * @param maplen the length of the map + * This function is used when creating the subset rearranger (it + * is not used for the box rearranger). It is called by get_regions(). + * + * Preconditions: + *
    + *
  • ndims is > 0 + *
  • maplen is > 0 + *
  • All elements of map are inside the bounds specified by gdimlen. + *
+ * + * Note that the map array is 1 based, but calculations are 0 based. + * + * @param ndims the number of dimensions. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. + * @param maplen the length of the map. * @param map - * @param start array of start indicies - * @param count array of counts - * @returns 0 on success, error code otherwise. + * @param start array (length ndims) that will get start indicies of + * found region. + * @param count array (length ndims) that will get counts of found + * region. + * @returns length of the region found. */ -PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map, +PIO_Offset find_region(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, PIO_Offset *start, PIO_Offset *count) { - int dim; - int max_size[ndims]; PIO_Offset regionlen = 1; - /* Preconditions (which might be useful to check/assert): - ndims is > 0 - maplen is > 0 - all elements of map are inside the bounds specified by gdims - The map array is 1 based, but calculations are 0 based */ - idx_to_dim_list(ndims, gdims, map[0] - 1, start); + /* Check inputs. */ + pioassert(ndims > 0 && gdimlen && maplen > 0 && map && start && count, + "invalid input", __FILE__, __LINE__); + LOG((2, "find_region ndims = %d maplen = %d", ndims, maplen)); + + int max_size[ndims]; + + /* Convert the index which is the first element of map into global + * data space. */ + idx_to_dim_list(ndims, gdimlen, map[0] - 1, start); - /* Can't expand beyond the array edge.*/ - for (dim = 0; dim < ndims; ++dim) - max_size[dim] = gdims[dim] - start[dim]; + /* Can't expand beyond the array edge. Set up max_size array for + * expand_region call below. */ + for (int dim = 0; dim < ndims; ++dim) + { + max_size[dim] = gdimlen[dim] - start[dim]; + LOG((3, "max_size[%d] = %d", max_size[dim])); + } /* For each dimension, figure out how far we can expand in that dimension while staying contiguous in the input array. Start with the innermost dimension (ndims-1), and it will recurse through to the outermost dimensions. */ - expand_region(ndims - 1, gdims, maplen, map, 1, 1, max_size, count); + expand_region(ndims - 1, gdimlen, maplen, map, 1, 1, max_size, count); - for (dim = 0; dim < ndims; dim++) + /* Calculate the number of data elements in this region. */ + for (int dim = 0; dim < ndims; dim++) regionlen *= count[dim]; - return(regionlen); + return regionlen; } /** * Convert a global coordinate value into a local array index. * - * @param ndims the number of dimensions - * @param lcoord pointer to an offset - * @param count array of counts - * @returns 0 on success, error code otherwise. + * @param ndims the number of dimensions. + * @param lcoord pointer to an offset. + * @param count array of counts. + * @returns the local array index. */ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count) { PIO_Offset lindex = 0; PIO_Offset stride = 1; + /* Check inputs. */ + pioassert(ndims > 0 && lcoord && count, "invalid input", __FILE__, __LINE__); + for (int i = ndims - 1; i >= 0; i--) { lindex += lcoord[i] * stride; @@ -184,7 +206,10 @@ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset } /** - * Compute the max io buffersize needed for a given variable + * Compute the max io buffer size needed for an iodesc. It is the + * combined size (in number of data elements) of all the regions of + * data stored in the buffer of this iodesc. The max size is then set + * in the iodesc. * * @param io_comm the IO communicator * @param iodesc a pointer to the io_desc_t struct. @@ -192,38 +217,34 @@ PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset */ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc) { - PIO_Offset iosize, totiosize; - int i; - io_region *region; + PIO_Offset totiosize = 0; int mpierr; /* Return code from MPI calls. */ - assert(iodesc); + pioassert(iodesc, "need iodesc", __FILE__, __LINE__); /* compute the max io buffer size, for conveneance it is the * combined size of all regions */ - totiosize = 0; - region = iodesc->firstregion; - while(region) + for (io_region *region = iodesc->firstregion; region; region = region->next) { if (region->count[0] > 0) { - iosize = 1; - for (i = 0; i < iodesc->ndims; i++) + PIO_Offset iosize = 1; + for (int i = 0; i < iodesc->ndims; i++) iosize *= region->count[i]; totiosize += iosize; } - region = region->next; } + LOG((2, "compute_maxIObuffersize got totiosize = %lld", totiosize)); /* Share the max io buffer size with all io tasks. */ if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); + pioassert(totiosize > 0, "totiosize <= 0", __FILE__, __LINE__); + LOG((2, "after allreduce compute_maxIObuffersize got totiosize = %lld", totiosize)); + /* Remember the result. */ iodesc->maxiobuflen = totiosize; - if (iodesc->maxiobuflen <= 0) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - return PIO_NOERR; } @@ -231,41 +252,47 @@ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc) * Create the derived MPI datatypes used for comp2io and io2comp * transfers. Used in define_iodesc_datatypes(). * - * @param basetype The type of data (int,real,double). - * @param msgcnt The number of MPI messages/tasks to use. - * @param dlen The length of the data array. - * @param mindex An array of indexes into the data array from the comp - * map - * @param mcount The number of indexes to be put on each mpi - * message/task + * @param basetype The MPI type of data (MPI_INT, etc.). + * @param msgcnt This is the number of MPI types that are created. + * @param mindex An array (length numinds) of indexes into the data + * array from the comp map. Will be NULL when count is zero. + * @param mcount An array (length msgcnt) with the number of indexes + * to be put on each mpi message/task. * @param mfrom A pointer to the previous structure in the read/write - * list - * @param mtype The final data structure sent through MPI to the - * read/write + * list. This is always NULL for the BOX rearranger. + * @param mtype pointer to an array (length msgcnt) which gets the + * created datatypes. Will be NULL when iodesc->nrecvs == 0. * @returns 0 on success, error code otherwise. */ -int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, +int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, const PIO_Offset *mindex, const int *mcount, int *mfrom, MPI_Datatype *mtype) { - PIO_Offset bsizeT[msgcnt]; int blocksize; int numinds = 0; PIO_Offset *lindex = NULL; int mpierr; /* Return code from MPI functions. */ + /* Check inputs. */ + pioassert(msgcnt > 0 && mcount, "invalid input", __FILE__, __LINE__); + + PIO_Offset bsizeT[msgcnt]; + + LOG((1, "create_mpi_datatypes basetype = %d msgcnt = %d", basetype, msgcnt)); + LOG((2, "MPI_BYTE = %d MPI_CHAR = %d MPI_SHORT = %d MPI_INT = %d MPI_DOUBLE = %d", + MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_DOUBLE)); + + /* How many indicies in the array? */ for (int j = 0; j < msgcnt; j++) numinds += mcount[j]; - - pioassert(dlen >= 0, "dlen < 0", __FILE__, __LINE__); - pioassert(numinds >= 0, "num inds < 0", __FILE__, __LINE__); + LOG((2, "numinds = %d", numinds)); if (mindex) { - /* memcpy(lindex, mindex, (size_t) (dlen*sizeof(PIO_Offset)));*/ if (!(lindex = malloc(numinds * sizeof(PIO_Offset)))) - return PIO_ENOMEM; + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset))); + LOG((3, "allocated lindex, copied mindex")); } bsizeT[0] = 0; @@ -273,82 +300,100 @@ int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, PIO_Offset dlen, int pos = 0; int ii = 0; - /* If there are no messages don't need to create any datatypes. */ - if (msgcnt > 0) + if (mfrom == NULL) { - if (mfrom == NULL) + LOG((3, "mfrom is NULL")); + for (int i = 0; i < msgcnt; i++) { - for (int i = 0; i < msgcnt; i++) + if (mcount[i] > 0) { - if (mcount[i] > 0) - { - bsizeT[ii] = GCDblocksize(mcount[i], lindex+pos); - ii++; - pos += mcount[i]; - } + /* Look for the largest block of data for io which + * can be expressed in terms of start and + * count. */ + bsizeT[ii] = GCDblocksize(mcount[i], lindex + pos); + ii++; + pos += mcount[i]; } - blocksize = (int)lgcd_array(ii ,bsizeT); - } - else - { - blocksize=1; } + blocksize = (int)lgcd_array(ii, bsizeT); + } + else + { + blocksize = 1; + } + LOG((3, "blocksize = %d", blocksize)); - /* pos is an index to the start of each message block. */ - pos = 0; - for (int i = 0; i < msgcnt; i++) + /* pos is an index to the start of each message block. */ + pos = 0; + for (int i = 0; i < msgcnt; i++) + { + if (mcount[i] > 0) { - if (mcount[i] > 0) + int len = mcount[i] / blocksize; + int displace[len]; + if (blocksize == 1) { - int len = mcount[i] / blocksize; - int displace[len]; - if (blocksize == 1) + if (!mfrom) { - if (!mfrom) - { - for (int j = 0; j < len; j++) - displace[j] = (int)(lindex[pos + j]); - } - else - { - int k = 0; - for (int j = 0; j < numinds; j++) - if (mfrom[j] == i) - displace[k++] = (int)(lindex[j]); - } - + for (int j = 0; j < len; j++) + displace[j] = (int)(lindex[pos + j]); } else { - for (int j = 0; j < mcount[i]; j++) - (lindex + pos)[j]++; - - for (int j = 0; j < len; j++) - displace[j] = ((lindex + pos)[j * blocksize] - 1); + int k = 0; + for (int j = 0; j < numinds; j++) + if (mfrom[j] == i) + displace[k++] = (int)(lindex[j]); } - if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace, basetype, mtype + i))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if (mtype[i] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + } + else + { + for (int j = 0; j < mcount[i]; j++) + (lindex + pos)[j]++; - /* Commit the MPI data type. */ - if ((mpierr = MPI_Type_commit(mtype + i))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - pos += mcount[i]; + for (int j = 0; j < len; j++) + displace[j] = ((lindex + pos)[j * blocksize] - 1); } + +#if PIO_ENABLE_LOGGING + for (int j = 0; j < len; j++) + LOG((3, "displace[%d] = %d", j, displace[j])); +#endif /* PIO_ENABLE_LOGGING */ + + LOG((3, "calling MPI_Type_create_indexed_block len = %d blocksize = %d " + "basetype = %d", len, blocksize, basetype)); + /* Create an indexed datatype with constant-sized blocks. */ + if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace, + basetype, &mtype[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + + if (mtype[i] == PIO_DATATYPE_NULL) + return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + + /* Commit the MPI data type. */ + LOG((3, "about to commit type")); + if ((mpierr = MPI_Type_commit(&mtype[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + pos += mcount[i]; } - if (lindex) - free(lindex); } + /* Free resources. */ + if (lindex) + free(lindex); + + LOG((3, "done with create_mpi_datatypes()")); return PIO_NOERR; } /** - * Create the derived MPI datatypes used for comp2io and io2comp - * transfers. + * If needed, create the derived MPI datatypes used for comp2io and + * io2comp transfers. + * + * If iodesc->stype and iodesc->rtype arrays already exist, this + * function does nothing. This function is called from + * rearrange_io2comp() and rearrange_comp2io(). * * NOTE from Jim: I am always oriented toward write so recieve * always means io tasks and send always means comp tasks. The @@ -366,142 +411,140 @@ int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc) { int ret; /* Return value. */ + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + LOG((1, "define_iodesc_datatypes ios->ioproc = %d", ios->ioproc)); + /* Set up the to transfer data to and from the IO tasks. */ if (ios->ioproc) { + /* If the types for the IO tasks have not been created, then + * create them. */ if (!iodesc->rtype) { if (iodesc->nrecvs > 0) { - /* Allocate memory for array of data. */ + /* Allocate memory for array of MPI types for the IO tasks. */ if (!(iodesc->rtype = malloc(iodesc->nrecvs * sizeof(MPI_Datatype)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "allocated memory for IO task MPI types iodesc->nrecvs = %d " + "iodesc->rearranger = %d", iodesc->nrecvs, iodesc->rearranger)); /* Initialize data types to NULL. */ for (int i = 0; i < iodesc->nrecvs; i++) iodesc->rtype[i] = PIO_DATATYPE_NULL; - /* Create the datatypes, which will be used both to - * receive and to send data. */ - if (iodesc->rearranger == PIO_REARR_SUBSET) - { - if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, - iodesc->rindex, iodesc->rcount, iodesc->rfrom, - iodesc->rtype))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - } - else - { - if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->llen, - iodesc->rindex, iodesc->rcount, NULL, iodesc->rtype))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - } + /* The different rearrangers get different values for mfrom. */ + int *mfrom = iodesc->rearranger == PIO_REARR_SUBSET ? iodesc->rfrom : NULL; + + /* Create the MPI datatypes. */ + if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->rindex, + iodesc->rcount, mfrom, iodesc->rtype))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); } } } - /* Define the send datatypes if they don't exist. */ + /* Define the datatypes for the computation components if they + * don't exist. (These will be the send side in a write + * operation.) */ if (!iodesc->stype) { int ntypes; - int ncnt; - if (iodesc->rearranger == PIO_REARR_SUBSET) - { - /* Subset rearranger gets one type. */ - ntypes = 1; - ncnt = iodesc->scount[0]; - } - else - { - /* Box rearranger gets one type per IO task. */ - ntypes = ios->num_iotasks; - ncnt = iodesc->ndof; - } + /* Subset rearranger gets one type; box rearranger gets one + * type per IO task. */ + ntypes = iodesc->rearranger == PIO_REARR_SUBSET ? 1 : ios->num_iotasks; - /* Allocate memory for array of send types. */ + /* Allocate memory for array of MPI types for the computation tasks. */ if (!(iodesc->stype = malloc(ntypes * sizeof(MPI_Datatype)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated memory for computation MPI types ntypes = %d", ntypes)); /* Initialize send types to NULL. */ for (int i = 0; i < ntypes; i++) iodesc->stype[i] = PIO_DATATYPE_NULL; + /* Remember how many types we created for the send side. */ iodesc->num_stypes = ntypes; /* Create the MPI data types. */ - if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, ncnt, iodesc->sindex, + LOG((3, "about to call create_mpi_datatypes for computation MPI types")); + if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, iodesc->sindex, iodesc->scount, NULL, iodesc->stype))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); } + LOG((3, "done with define_iodesc_datatypes()")); return PIO_NOERR; } /** - * Completes the mapping for the box rearranger. + * Completes the mapping for the box rearranger. This function is + * called from box_rearrange_create(). It is not used for the subset + * rearranger. + * + * This function: + *
    + *
  • Allocates and inits iodesc->scount, an array (length + * ios->num_iotasks) containing number of data elements sent to each + * IO task from current compute task. + *
  • Uses pio_swapm() to send iodesc->scount array from each + * computation task to all IO tasks. + *
  • On IO tasks, allocates and inits iodesc->rcount and + * iodesc->rfrom arrays (length max(1, nrecvs)) which holds the amount + * of data to expect from each compute task and the rank of that + * task. . + *
  • Allocates and inits iodesc->sindex arrays (length iodesc->ndof) + * which holds indecies for computation tasks. + *
  • On IO tasks, allocates and init iodesc->rindex (length + * totalrecv) with indices of the data to be sent/received from this + * io task to each compute task. + *
  • Uses pio_swapm() to send list of indicies on each compute task + * to the IO tasks. + *
* - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param maplen the length of the map - * @param dest_ioproc an array of IO task numbers - * @param dest_ioindex - * @param mycomm an MPI communicator + * @param dest_ioproc an array (length maplen) of IO task numbers. + * @param dest_ioindex an array (length maplen) of IO indicies. * @returns 0 on success, error code otherwise. */ -int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, - const int *dest_ioproc, const PIO_Offset *dest_ioindex, MPI_Comm mycomm) +int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, + const int *dest_ioproc, const PIO_Offset *dest_ioindex) { - int i; - int iorank; - int rank; - int ntasks; - int mpierr; /* Return call from MPI functions. */ - - /* Find size of communicator, and task rank. */ - if ((mpierr = MPI_Comm_rank(mycomm, &rank))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - MPI_Datatype sr_types[ntasks]; - int send_counts[ntasks]; - int send_displs[ntasks]; - int recv_counts[ntasks]; - int recv_displs[ntasks]; int *recv_buf = NULL; - int nrecvs; + int nrecvs = 0; int ierr; - int io_comprank; - int ioindex; - int tsize; - int numiotasks; - PIO_Offset s2rindex[iodesc->ndof]; - pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__); + /* Check inputs. */ + pioassert(ios && iodesc && dest_ioproc && dest_ioindex && + iodesc->rearranger == PIO_REARR_BOX && ios->num_uniontasks > 0, + "invalid input", __FILE__, __LINE__); + LOG((1, "compute_counts ios->num_uniontasks = %d", ios->num_uniontasks)); - /* Subset rearranger always gets 1 IO task. */ - if (iodesc->rearranger == PIO_REARR_BOX) - numiotasks = ios->num_iotasks; - else - numiotasks = 1; + /* Arrays for swapm all to all gather calls. */ + MPI_Datatype sr_types[ios->num_uniontasks]; + int send_counts[ios->num_uniontasks]; + int send_displs[ios->num_uniontasks]; + int recv_counts[ios->num_uniontasks]; + int recv_displs[ios->num_uniontasks]; - /* Allocate memory for the array of counts. */ - if (!(iodesc->scount = malloc(numiotasks * sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* The list of indeces on each compute task */ + PIO_Offset s2rindex[iodesc->ndof]; - /* Initialize counts to zero. */ - for (i = 0; i < numiotasks; i++) - iodesc->scount[i] = 0; + /* Allocate memory for the array of counts and init to zero. */ + if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - /* iodesc->scount is the amount of data sent to each task from the - * current task */ - for (i = 0; i < maplen; i++) + /* iodesc->scount is the number of data elements sent to each IO + * task from the current compute task. dest_ioindex[i] may be + * -1. */ + for (int i = 0; i < iodesc->ndof; i++) if (dest_ioindex[i] >= 0) (iodesc->scount[dest_ioproc[i]])++; - /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + /* Initialize arrays used in swapm call. */ + for (int i = 0; i < ios->num_uniontasks; i++) { send_counts[i] = 0; send_displs[i] = 0; @@ -510,66 +553,74 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, sr_types[i] = MPI_INT; } - /* ??? */ - for (i = 0; i < numiotasks; i++) + /* Setup for the swapm call. iodesc->scount is the amount of data + * this compute task will transfer to/from each iotask. For the + * box rearranger there can be more than one IO task per compute + * task. This provides enough information to know the size of data + * on the iotask, so at line 557 we allocate arrays to hold the + * map on the iotasks. iodesc->rcount is an array of the amount of + * data to expect from each compute task and iodesc->rfrom is the + * rank of that task. */ + if (ios->compproc) { - int io_comprank; - if (iodesc->rearranger == PIO_REARR_SUBSET) - io_comprank = 0; - else - io_comprank = ios->ioranks[i]; - send_counts[io_comprank] = 1; - send_displs[io_comprank] = i * sizeof(int); + for (int i = 0; i < ios->num_iotasks; i++) + { + send_counts[ios->ioranks[i]] = 1; + send_displs[ios->ioranks[i]] = i * sizeof(int); + LOG((3, "send_counts[%d] = %d send_displs[%d] = %d", ios->ioranks[i], + send_counts[ios->ioranks[i]], ios->ioranks[i], send_displs[ios->ioranks[i]])); + } } - - /* ??? */ + + /* IO tasks need to know how many data elements they will receive + * from each compute task. Allocate space for that, and set up + * swapm call. */ if (ios->ioproc) { - /* Allocate memory to hold array of tasks that have recieved - * data ??? */ - if (!(recv_buf = calloc(ntasks, sizeof(int)))) + /* Allocate memory to hold array of the scounts from all + * computation tasks. */ + if (!(recv_buf = calloc(ios->num_comptasks, sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - /* Initialize arrays that keep track of receives. */ - for (i = 0; i < ntasks; i++) + /* Initialize arrays that keep track of ???. */ + for (int i = 0; i < ios->num_comptasks; i++) { - recv_counts[i] = 1; - recv_displs[i] = i * sizeof(int); + recv_counts[ios->compranks[i]] = 1; + recv_displs[ios->compranks[i]] = i * sizeof(int); } } - /* Share the iodesc->scount from each compute task to all io tasks. */ - ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types, - recv_buf, recv_counts, recv_displs, sr_types, - mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); + LOG((2, "about to share scount from each compute task to all IO tasks.")); + /* Share the iodesc->scount from each compute task to all IO + * tasks. The scounts will end up in array recv_buf. */ + if ((ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types, + recv_buf, recv_counts, recv_displs, sr_types, ios->union_comm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); - /* ??? */ - nrecvs = 0; + /* On IO tasks, set up data receives. */ if (ios->ioproc) { - for (i = 0; i < ntasks; i++) + /* Count the number of non-zero scounts from the compute + * tasks. */ + for (int i = 0; i < ios->num_comptasks; i++) + { if (recv_buf[i] != 0) nrecvs++; + LOG((3, "recv_buf[%d] = %d", i, recv_buf[i])); + } /* Get memory to hold the count of data receives. */ - if (!(iodesc->rcount = malloc(max(1, nrecvs) * sizeof(int)))) + if (!(iodesc->rcount = calloc(max(1, nrecvs), sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Get memory to hold the list of task data was from. */ - if (!(iodesc->rfrom = malloc(max(1, nrecvs) * sizeof(int)))) + if (!(iodesc->rfrom = calloc(max(1, nrecvs), sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - for (i = 0; i < max(1, nrecvs); i++) - { - iodesc->rcount[i] = 0; - iodesc->rfrom[i] = 0; - } + LOG((3, "allocared rfrom max(1, nrecvs) = %d", max(1, nrecvs))); nrecvs = 0; - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ios->num_comptasks; i++) { if (recv_buf[i] != 0) { @@ -583,39 +634,50 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, /* ??? */ iodesc->nrecvs = nrecvs; + LOG((3, "iodesc->nrecvs = %d", iodesc->nrecvs)); + + /* Allocate an array for indicies on the computation tasks (the + * send side when writing). */ if (iodesc->sindex == NULL && iodesc->ndof > 0) if (!(iodesc->sindex = malloc(iodesc->ndof * sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((2, "iodesc->ndof = %d", iodesc->ndof)); - int tempcount[numiotasks]; - int spos[numiotasks]; + int tempcount[ios->num_iotasks]; + int spos[ios->num_iotasks]; /* ??? */ spos[0] = 0; tempcount[0] = 0; - for (i = 1; i < numiotasks; i++) + for (int i = 1; i < ios->num_iotasks; i++) { spos[i] = spos[i - 1] + iodesc->scount[i - 1]; tempcount[i] = 0; + LOG((3, "spos[%d] = %d tempcount[%d] = %d", i, spos[i], i, tempcount[i])); } - for (i = 0; i < maplen; i++) + /* ??? */ + for (int i = 0; i < iodesc->ndof; i++) { + int iorank; + int ioindex; + iorank = dest_ioproc[i]; ioindex = dest_ioindex[i]; if (iorank > -1) { /* this should be moved to create_box */ - if (iodesc->rearranger == PIO_REARR_BOX) - iodesc->sindex[spos[iorank] + tempcount[iorank]] = i; + iodesc->sindex[spos[iorank] + tempcount[iorank]] = i; s2rindex[spos[iorank] + tempcount[iorank]] = ioindex; (tempcount[iorank])++; + LOG((3, "iorank = %d ioindex = %d tempcount[iorank] = %d", iorank, ioindex, + tempcount[iorank])); } } /* Initialize arrays to zeros. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ios->num_uniontasks; i++) { send_counts[i] = 0; send_displs[i] = 0; @@ -623,97 +685,83 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int maplen, recv_displs[i] = 0; } - /* Find the size of the offset type. */ - if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - for (i = 0; i < ntasks; i++) - sr_types[i] = MPI_OFFSET; - - for (i = 0; i < numiotasks; i++) + /* ??? */ + for (int i = 0; i < ios->num_iotasks; i++) { - if (iodesc->rearranger == PIO_REARR_BOX) - io_comprank = ios->ioranks[i]; - else - io_comprank = 0; - - send_counts[io_comprank] = iodesc->scount[i]; - if (send_counts[io_comprank] > 0) - send_displs[io_comprank] = spos[i] * tsize; + /* Subset rearranger needs one type, box rearranger needs one for + * each IO task. */ + send_counts[ios->ioranks[i]] = iodesc->scount[i]; + if (send_counts[ios->ioranks[i]] > 0) + send_displs[ios->ioranks[i]] = spos[i] * SIZEOF_MPI_OFFSET; + LOG((3, "ios->ioranks[i] = %d iodesc->scount[%d] = %d spos[%d] = %d", + ios->ioranks[i], i, iodesc->scount[i], i, spos[i])); } + /* Only do this on IO tasks. */ if (ios->ioproc) { int totalrecv = 0; - for (i = 0; i < nrecvs; i++) + for (int i = 0; i < nrecvs; i++) { recv_counts[iodesc->rfrom[i]] = iodesc->rcount[i]; - totalrecv+=iodesc->rcount[i]; + totalrecv += iodesc->rcount[i]; } - recv_displs[0] = 0; - for (i = 1; i < nrecvs; i++) - recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i-1]] + iodesc->rcount[i-1] * tsize; + recv_displs[0] = 0; + for (int i = 1; i < nrecvs; i++) + { + recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i - 1]] + + iodesc->rcount[i - 1] * SIZEOF_MPI_OFFSET; + LOG((3, "iodesc->rfrom[%d] = %d recv_displs[iodesc->rfrom[i]] = %d", i, + iodesc->rfrom[i], recv_displs[iodesc->rfrom[i]])); + } + /* rindex is an array of the indices of the data to be sent from + this io task to each compute task. */ + LOG((3, "totalrecv = %d", totalrecv)); if (totalrecv > 0) { totalrecv = iodesc->llen; /* can reduce memory usage here */ if (!(iodesc->rindex = calloc(totalrecv, sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((3, "allocated totalrecv elements in rindex array")); } } + /* For the swapm call below, init the types to MPI_OFFSET. */ + for (int i = 0; i < ios->num_uniontasks; i++) + sr_types[i] = MPI_OFFSET; + + /* Here we are sending the mapping from the index on the compute + * task to the index on the io task. */ /* s2rindex is the list of indeces on each compute task */ - /* - printf("%d s2rindex: ", ios->comp_rank); - for (i = 0;indof;i++) - printf("%ld ",s2rindex[i]); - printf("\n"); - */ - ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex, - recv_counts, recv_displs, sr_types, mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); - - /* rindex is an array of the indices of the data to be sent from - this io task to each compute task. */ - /* - if (ios->ioproc){ - printf("%d rindex: ",ios->io_rank); - for (int j = 0;jllen;j++) - printf(" %ld ",iodesc->rindex[j]); - printf("\n"); - } - */ - return ierr; + LOG((3, "sending mapping")); + if ((ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex, + recv_counts, recv_displs, sr_types, ios->union_comm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + + return PIO_NOERR; } /** - * Moves data from compute tasks to IO tasks. + * Moves data from compute tasks to IO tasks. This is called from + * PIOc_write_darray_multi(). * - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param sbuf send buffer. - * @param rbuf receive buffer. + * @param sbuf send buffer. May be NULL. + * @param rbuf receive buffer. May be NULL. * @param nvars number of variables. * @returns 0 on success, error code otherwise. */ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf, int nvars) { - int ntasks; - int niotasks; - int *scount = iodesc->scount; - int i, tsize; - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - MPI_Comm mycomm; - int mpierr; /* Return code from MPI calls. */ + int ntasks; /* Number of tasks in communicator. */ + int niotasks; /* Number of IO tasks. */ + MPI_Comm mycomm; /* Communicator that data is transferred over. */ + int mpierr; /* Return code from MPI calls. */ int ret; #ifdef TIMING @@ -721,11 +769,12 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, #endif /* Caller must provide these. */ - pioassert(iodesc && nvars > 0, "invalid input", __FILE__, __LINE__); + pioassert(ios && iodesc && nvars > 0, "invalid input", __FILE__, __LINE__); - LOG((2, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars, + LOG((1, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars, iodesc->rearranger)); + /* Different rearraangers use different communicators. */ if (iodesc->rearranger == PIO_REARR_BOX) { mycomm = ios->union_comm; @@ -741,83 +790,89 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - /* Get the size of the MPI type. */ - if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - LOG((3, "ntasks = %d tsize = %d", ntasks, tsize)); - - /* Define the MPI data types that will be used for this - * io_desc_t. */ - if ((ret = define_iodesc_datatypes(ios, iodesc))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - - /* Allocate arrays needed by the pio_swapm() function. */ - if (!(sendcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(rdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* These are parameters to pio_swapm to send data from compute to + * IO tasks. */ + int sendcounts[ntasks]; + int recvcounts[ntasks]; + int sdispls[ntasks]; + int rdispls[ntasks]; + MPI_Datatype sendtypes[ntasks]; + MPI_Datatype recvtypes[ntasks]; - /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + /* Initialize pio_swapm parameter arrays. */ + for (int i = 0; i < ntasks; i++) { + sendcounts[i] = 0; + recvcounts[i] = 0; + sdispls[i] = 0; + rdispls[i] = 0; recvtypes[i] = PIO_DATATYPE_NULL; sendtypes[i] = PIO_DATATYPE_NULL; } + LOG((3, "ntasks = %d iodesc->basetype_size = %d niotasks = %d", ntasks, + iodesc->basetype_size, niotasks)); + + /* If it has not already been done, define the MPI data types that + * will be used for this io_desc_t. */ + if ((ret = define_iodesc_datatypes(ios, iodesc))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* If this io proc will exchange data with compute tasks create a * MPI DataType for that exchange. */ + LOG((2, "ios->ioproc %d iodesc->nrecvs = %d", ios->ioproc, iodesc->nrecvs)); if (ios->ioproc && iodesc->nrecvs > 0) { - for (i = 0; i < iodesc->nrecvs; i++) + for (int i = 0; i < iodesc->nrecvs; i++) { if (iodesc->rtype[i] != PIO_DATATYPE_NULL) { + LOG((3, "iodesc->rtype[%d] = %d iodesc->rearranger = %d", i, iodesc->rtype[i], + iodesc->rearranger)); if (iodesc->rearranger == PIO_REARR_SUBSET) { - recvcounts[ i ] = 1; - - /* The stride here is the length of the collected array (llen) */ - - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint) iodesc->llen * tsize, iodesc->rtype[i], - recvtypes + i))) + LOG((3, "exchanging data for subset rearranger")); + recvcounts[i] = 1; + + /* Create an MPI derived data type from equally + * spaced blocks of the same size. The block size + * is 1, the stride here is the length of the + * collected array (llen). */ +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - if (recvtypes[i] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(recvtypes + i))) +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[i] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); - /*recvtypes[ i ] = iodesc->rtype[i]; */ + if ((mpierr = MPI_Type_commit(&recvtypes[i]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); } else { + LOG((3, "exchanging data for box rearranger")); + LOG((3, "i = %d iodesc->rfrom[i] = %d recvcounts[iodesc->rfrom[i]] = %d", i, + iodesc->rfrom[i], recvcounts[iodesc->rfrom[i]])); recvcounts[iodesc->rfrom[i]] = 1; - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * tsize, iodesc->rtype[i], - recvtypes + iodesc->rfrom[i]))) +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size, + iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[iodesc->rfrom[i]] != PIO_DATATYPE_NULL, "bad mpi type", + __FILE__, __LINE__); - if (recvtypes[iodesc->rfrom[i]] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(recvtypes+iodesc->rfrom[i]))) + if ((mpierr = MPI_Type_commit(&recvtypes[iodesc->rfrom[i]]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - /* recvtypes[ iodesc->rfrom[i] ] = iodesc->rtype[i]; */ rdispls[iodesc->rfrom[i]] = 0; } } @@ -826,57 +881,57 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, /* On compute tasks loop over iotasks and create a data type for * each exchange. */ - for (i = 0; i < niotasks; i++) + for (int i = 0; i < niotasks; i++) { int io_comprank = ios->ioranks[i]; if (iodesc->rearranger == PIO_REARR_SUBSET) - io_comprank=0; + io_comprank = 0; - if (scount[i] > 0 && sbuf != NULL) + LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i])); + if (iodesc->scount[i] > 0 && sbuf) { + LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank)); sendcounts[io_comprank] = 1; - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * tsize, iodesc->stype[i], - sendtypes + io_comprank))) +#if PIO_USE_MPISERIAL + if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size, + iodesc->stype[i], &sendtypes[io_comprank]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#else + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size, + iodesc->stype[i], &sendtypes[io_comprank]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); +#endif /* PIO_USE_MPISERIAL */ + pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); - if (sendtypes[io_comprank] == PIO_DATATYPE_NULL) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(sendtypes + io_comprank))) + if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } else { - sendcounts[io_comprank]=0; + sendcounts[io_comprank] = 0; } } + /* Data in sbuf on the compute nodes is sent to rbuf on the ionodes */ - pio_swapm(sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, mycomm, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs, - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_isend, - iodesc->rearr_opts.comm_fc_opts_comp2io.max_pend_req); + LOG((2, "about to call pio_swapm for sbuf")); + if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, + rbuf, recvcounts, rdispls, recvtypes, mycomm, + &iodesc->rearr_opts.comp2io))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Free the MPI types. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ntasks; i++) { + LOG((3, "freeing MPI types for task %d", i)); if (sendtypes[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(sendtypes + i))) + if ((mpierr = MPI_Type_free(&sendtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); if (recvtypes[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(recvtypes + i))) + if ((mpierr = MPI_Type_free(&recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } - /* Free memory. */ - free(sendcounts); - free(recvcounts); - free(sdispls); - free(rdispls); - free(sendtypes); - free(recvtypes); - #ifdef TIMING GPTLstop("PIO:rearrange_comp2io"); #endif @@ -885,7 +940,8 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /** - * Moves data from IO tasks to compute tasks. + * Moves data from IO tasks to compute tasks. This function is used in + * PIOc_read_darray(). * * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. @@ -899,18 +955,11 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, MPI_Comm mycomm; int ntasks; int niotasks; - int *scount = iodesc->scount; - int *sendcounts; - int *recvcounts; - int *sdispls; - int *rdispls; - MPI_Datatype *sendtypes; - MPI_Datatype *recvtypes; - int i; int mpierr; /* Return code from MPI calls. */ int ret; - assert(iodesc); + /* Check inputs. */ + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); #ifdef TIMING GPTLstart("PIO:rearrange_io2comp"); @@ -926,8 +975,9 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, else { mycomm = iodesc->subset_comm; - niotasks=1; + niotasks = 1; } + LOG((3, "niotasks = %d", niotasks)); /* Get the size of this communicator. */ if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) @@ -939,34 +989,30 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Allocate arrays needed by the pio_swapm() function. */ - if (!(sendcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvcounts = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(rdispls = calloc(ntasks, sizeof(int)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(sendtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - - if (!(recvtypes = malloc(ntasks * sizeof(MPI_Datatype)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + int sendcounts[ntasks]; + int recvcounts[ntasks]; + int sdispls[ntasks]; + int rdispls[ntasks]; + MPI_Datatype sendtypes[ntasks]; + MPI_Datatype recvtypes[ntasks]; /* Initialize arrays. */ - for (i = 0; i < ntasks; i++) + for (int i = 0; i < ntasks; i++) { + sendcounts[i] = 0; + recvcounts[i] = 0; + sdispls[i] = 0; + rdispls[i] = 0; sendtypes[i] = PIO_DATATYPE_NULL; recvtypes[i] = PIO_DATATYPE_NULL; } - /* In IO tasks ??? */ + /* In IO tasks set up sendcounts/sendtypes for pio_swapm() call + * below. */ if (ios->ioproc) - for (i = 0; i < iodesc->nrecvs; i++) + { + for (int i = 0; i < iodesc->nrecvs; i++) + { if (iodesc->rtype[i] != PIO_DATATYPE_NULL) { if (iodesc->rearranger == PIO_REARR_SUBSET) @@ -983,18 +1029,21 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, sendtypes[iodesc->rfrom[i]] = iodesc->rtype[i]; } } + } + } /* In the box rearranger each comp task may communicate with * multiple IO tasks here we are setting the count and data type * of the communication of a given compute task with each io * task. */ - for (i = 0; i < niotasks; i++) + for (int i = 0; i < niotasks; i++) { int io_comprank = ios->ioranks[i]; + if (iodesc->rearranger == PIO_REARR_SUBSET) io_comprank = 0; - if (scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL) + if (iodesc->scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL) { recvcounts[io_comprank] = 1; recvtypes[io_comprank] = iodesc->stype[i]; @@ -1002,19 +1051,9 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */ - pio_swapm(sbuf, sendcounts, sdispls, sendtypes, - rbuf, recvcounts, rdispls, recvtypes, mycomm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - /* Release memory. */ - free(sendcounts); - free(recvcounts); - free(sdispls); - free(rdispls); - free(sendtypes); - free(recvtypes); + if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, + rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); #ifdef TIMING GPTLstop("PIO:rearrange_io2comp"); @@ -1024,44 +1063,53 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } /** - * Determine fill value. + * Determine whether fill values are needed. This function compares + * how much data we have to how much data is in a record (or + * non-record var). If we have enough data to completely fill the + * variable, then fill is not needed. * - * @param ios pointer to the iosystem_desc_t struct + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. - * @param gsize pointer to an array of sizes - * @param compmap + * @param gdimlen pointer to an array length iodesc->ndims with the + * global array sizes for one record (for record vars) or for the + * entire var (for non-record vars). + * @param compmap only used for the box communicator. * @returns 0 on success, error code otherwise. */ -int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, +int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gdimlen, const PIO_Offset *compmap) { PIO_Offset totalllen = 0; PIO_Offset totalgridsize = 1; - int i; int mpierr; /* Return code from MPI calls. */ - assert(iodesc); + /* Check inputs. */ + pioassert(ios && iodesc && gdimlen && compmap, "invalid input", + __FILE__, __LINE__); - for (i = 0; i < iodesc->ndims; i++) - totalgridsize *= gsize[i]; + /* Determine size of data space. */ + for (int i = 0; i < iodesc->ndims; i++) + totalgridsize *= gdimlen[i]; + /* Determine how many values we have locally. */ if (iodesc->rearranger == PIO_REARR_SUBSET) - totalllen=iodesc->llen; + totalllen = iodesc->llen; else - for (i = 0; indof; i++) + for (int i = 0; i < iodesc->ndof; i++) if (compmap[i] > 0) totalllen++; + /* Add results accross communicator. */ + LOG((2, "determine_fill before allreduce totalllen = %d totalgridsize = %d", + totalllen, totalgridsize)); if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM, ios->union_comm))) check_mpi(NULL, mpierr, __FILE__, __LINE__); + LOG((2, "after allreduce totalllen = %d", totalllen)); /* If the total size of the data provided to be written is < the * total data size then we need fill values. */ - if (totalllen < totalgridsize) - iodesc->needsfill = true; - else - iodesc->needsfill = false; + iodesc->needsfill = totalllen < totalgridsize; /* TURN OFF FILL for timing test iodesc->needsfill=false; */ @@ -1069,174 +1117,214 @@ int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize, return PIO_NOERR; } -/** - * Prints the IO desc information to stdout. - * - * @param iodesc a pointer to the io_desc_t struct. - * @returns 0 on success, error code otherwise. - */ -void iodesc_dump(io_desc_t *iodesc) -{ - assert(iodesc); - - printf("ioid= %d\n", iodesc->ioid); -/* printf("async_id= %d\n",iodesc->async_id);*/ - printf("nrecvs= %d\n", iodesc->nrecvs); - printf("ndof= %d\n", iodesc->ndof); - printf("ndims= %d\n", iodesc->ndims); - printf("num_aiotasks= %d\n", iodesc->num_aiotasks); - printf("rearranger= %d\n", iodesc->rearranger); - printf("maxregions= %d\n", iodesc->maxregions); - printf("needsfill= %d\n", (int)iodesc->needsfill); - - printf("llen= %lld\n", iodesc->llen); - printf("maxiobuflen= %d\n", iodesc->maxiobuflen); - - printf("rindex= "); - for (int j = 0; j < iodesc->llen; j++) - printf(" %lld ", iodesc->rindex[j]); - printf("\n"); -} - /** * The box rearranger computes a mapping between IO tasks and compute - * tasks such that the data on io tasks can be written with a single - * call to the underlying netcdf library. This may involve an all to + * tasks such that the data on IO tasks can be written with a single + * call to the underlying netCDF library. This may involve an all to * all rearrangement in the mapping, but should minimize data movement - * in lower level libraries + * in lower level libraries. * - * @param ios pointer to the iosystem_desc_t struct - * @param maplen the length of the map - * @param compmap - * @param gsize pointer to an array of sizes - * @param ndims the number of dimensions - * @param iodesc a pointer to the io_desc_t struct. + * On each compute task the application program passes a compmap array + * of length ndof. This array describes the arrangement of data in + * memory on that compute task. + * + * These arrays are gathered and rearranged to the IO-tasks (which are + * sometimes collocated with compute tasks), each IO task contains + * data from the compmap of one or more compute tasks in the iomap + * array and the length of that array is llen. + * + * This function: + *
    + *
  • For IO tasks, determines llen. + *
  • Determine whether fill values will be needed. + *
  • Do an allgether of llen values into array iomaplen. + *
  • For each IO task, send starts/counts to all compute tasks. + *
  • Find dest_ioindex and dest_ioproc for each element in the map. + *
  • Call compute_counts(). + *
  • On IO tasks, compute the max IO buffer size. + *
  • Call compute_maxaggregate_bytes(). + *
+ * + * @param ios pointer to the iosystem_desc_t struct. + * @param maplen the length of the map. This is the number of data + * elements on the compute task. + * @param compmap a 1 based array of offsets into the global space. A + * 0 in this array indicates a value which should not be transfered. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. + * @param ndims the number of dimensions. + * @param iodesc a pointer to the io_desc_t struct, which must be + * allocated before this function is called. * @returns 0 on success, error code otherwise. */ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, - const int *gsize, int ndims, io_desc_t *iodesc) + const int *gdimlen, int ndims, io_desc_t *iodesc) { - int nprocs = ios->num_comptasks; - int nioprocs = ios->num_iotasks; - PIO_Offset gstride[ndims]; - PIO_Offset start[ndims], count[ndims]; - int tsize, i, j, k; - int dest_ioproc[maplen]; - PIO_Offset dest_ioindex[maplen]; - int sndlths[nprocs]; - int sdispls[nprocs]; - int recvlths[nprocs]; - int rdispls[nprocs]; - MPI_Datatype dtypes[nprocs]; - PIO_Offset iomaplen[nioprocs]; - int mpierr; /* Return code from MPI functions. */ int ret; - assert(iodesc); - + /* Check inputs. */ + pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc, + "invalid input", __FILE__, __LINE__); + LOG((1, "box_rearrange_create maplen = %d ndims = %d ios->num_comptasks = %d " + "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks)); + + /* Allocate arrays needed for this function. */ + int dest_ioproc[maplen]; /* Destination IO task for each data element on compute task. */ + PIO_Offset dest_ioindex[maplen]; /* Offset into IO task array for each data element. */ + int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */ + int sdispls[ios->num_uniontasks]; /* Send displacements for swapm. */ + int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */ + int rdispls[ios->num_uniontasks]; /* Receive displacements for swapm. */ + MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */ + PIO_Offset iomaplen[ios->num_iotasks]; /* Gets the llen of all IO tasks. */ + + /* This is the box rearranger. */ iodesc->rearranger = PIO_REARR_BOX; + /* Number of elements of data on compute node. */ iodesc->ndof = maplen; - gstride[ndims-1] = 1; - for (int i= ndims - 2; i >= 0; i--) - gstride[i] = gstride[i + 1] * gsize[i + 1]; - - if ((mpierr = MPI_Type_size(MPI_OFFSET, &tsize))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - for (i = 0; i < maplen; i++) + /* Initialize array values. */ + for (int i = 0; i < maplen; i++) { dest_ioproc[i] = -1; dest_ioindex[i] = -1; } - for (i = 0; i < nprocs; i++) + + /* Initialize arrays used in swapm. */ + for (int i = 0; i < ios->num_uniontasks; i++) { - sndlths[i] = 0; + sendcounts[i] = 0; sdispls[i] = 0; - recvlths[i] = 0; + recvcounts[i] = 0; rdispls[i] = 0; dtypes[i] = MPI_OFFSET; } - iodesc->llen = 0; + + /* For IO tasks, determine llen, the length of the data array on + * the IO task. For computation tasks, llen will remain at 0. Also + * set up arrays for the allgather which will give every IO task a + * complete list of llens for each IO task. */ + LOG((3, "ios->ioproc = %d ios->num_uniontasks = %d", ios->ioproc, + ios->num_uniontasks)); + pioassert(iodesc->llen == 0, "error", __FILE__, __LINE__); if (ios->ioproc) { - for (i = 0; i < nprocs; i++) - sndlths[i] = 1; - - /* llen here is the number that will be read on each io task */ + /* Set up send counts for sending llen in all to all + * gather. We are sending to all tasks, IO and computation. */ + for (int i = 0; i < ios->num_comptasks; i++) + sendcounts[ios->compranks[i]] = 1; + for (int i = 0; i < ios->num_iotasks; i++) + sendcounts[ios->ioranks[i]] = 1; + + /* Determine llen, the lenght of the data array on this IO + * node, by multipliying the counts in the + * iodesc->firstregion. */ iodesc->llen = 1; - for (i = 0; i < ndims; i++) + for (int i = 0; i < ndims; i++) + { iodesc->llen *= iodesc->firstregion->count[i]; + LOG((3, "iodesc->firstregion->start[%d] = %d iodesc->firstregion->count[%d] = %d", + i, iodesc->firstregion->start[i], i, iodesc->firstregion->count[i])); + } + LOG((2, "iodesc->llen = %d", iodesc->llen)); } - if ((ret = determine_fill(ios, iodesc, gsize, compmap))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* - if (ios->ioproc){ - for (i = 0; ifirstregion->start[i],iodesc->firstregion->count[i]); - } - printf("\n%s %d\n",__FILE__,__LINE__); - } - */ + /* Determine whether fill values will be needed. */ + if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((2, "iodesc->needsfill = %d ios->num_iotasks = %d", iodesc->needsfill, + ios->num_iotasks)); - for (i = 0; i < nioprocs; i++) + /* Set up receive counts and displacements to for an AllToAll + * gather of llen. */ + for (int i = 0; i < ios->num_iotasks; i++) { - int io_comprank = ios->ioranks[i]; - recvlths[io_comprank] = 1; - rdispls[io_comprank] = i * tsize; + recvcounts[ios->ioranks[i]] = 1; + rdispls[ios->ioranks[i]] = i * SIZEOF_MPI_OFFSET; + LOG((3, "i = %d ios->ioranks[%d] = %d recvcounts[%d] = %d rdispls[%d] = %d", + i, i, ios->ioranks[i], ios->ioranks[i], recvcounts[ios->ioranks[i]], + ios->ioranks[i], rdispls[ios->ioranks[i]])); } - /* The length of each iomap - iomaplen = calloc(nioprocs, sizeof(PIO_Offset)); */ - pio_swapm(&(iodesc->llen), sndlths, sdispls, dtypes, - iomaplen, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - for (i = 0; i < nioprocs; i++) + /* All-gather the llen to all tasks into array iomaplen. */ + LOG((3, "calling pio_swapm to allgather llen into array iomaplen, ndims = %d dtypes[0] = %d", + ndims, dtypes)); + if ((ret = pio_swapm(&iodesc->llen, sendcounts, sdispls, dtypes, iomaplen, recvcounts, + rdispls, dtypes, ios->union_comm, &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->llen = %d", iodesc->llen)); +#if PIO_ENABLE_LOGGING + for (int i = 0; i < ios->num_iotasks; i++) + LOG((3, "iomaplen[%d] = %d", i, iomaplen[i])); +#endif /* PIO_ENABLE_LOGGING */ + + /* For each IO task send starts/counts to all compute tasks. */ + for (int i = 0; i < ios->num_iotasks; i++) { + /* The ipmaplen contains the llen (number of data elements) + * for this IO task. */ + LOG((2, "iomaplen[%d] = %d", i, iomaplen[i])); + /* If there is data for this IO task, send start/count to all + * compute tasks. */ if (iomaplen[i] > 0) { - int io_comprank = ios->ioranks[i]; - for (j = 0; j < nprocs; j++) + PIO_Offset start[ndims]; + PIO_Offset count[ndims]; + + /* Set up send/recv parameters for all to all gather of + * counts and starts. */ + for (int j = 0; j < ios->num_uniontasks; j++) { - sndlths[j] = 0; + sendcounts[j] = 0; sdispls[j] = 0; rdispls[j] = 0; - recvlths[j] = 0; - if (ios->union_rank == io_comprank) - sndlths[j] = ndims; + recvcounts[j] = 0; + if (ios->union_rank == ios->ioranks[i]) + sendcounts[j] = ndims; } - recvlths[io_comprank] = ndims; - - /* The count from iotask i is sent to all compute tasks */ - pio_swapm(iodesc->firstregion->count, sndlths, sdispls, dtypes, - count, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - /* The start from iotask i is sent to all compute tasks. */ - pio_swapm(iodesc->firstregion->start, sndlths, sdispls, dtypes, - start, recvlths, rdispls, dtypes, - ios->union_comm, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_hs, - iodesc->rearr_opts.comm_fc_opts_io2comp.enable_isend, - iodesc->rearr_opts.comm_fc_opts_io2comp.max_pend_req); - - for (k = 0; k < maplen; k++) + recvcounts[ios->ioranks[i]] = ndims; + + /* The count array from iotask i is sent to all compute tasks. */ + LOG((3, "about to call pio_swapm with count from iotask %d ndims = %d", + i, ndims)); + if ((ret = pio_swapm(iodesc->firstregion->count, sendcounts, sdispls, dtypes, count, + recvcounts, rdispls, dtypes, ios->union_comm, + &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + + /* The start array from iotask i is sent to all compute tasks. */ + LOG((3, "about to call pio_swapm with start from iotask %d ndims = %d", + i, ndims)); + if ((ret = pio_swapm(iodesc->firstregion->start, sendcounts, sdispls, dtypes, + start, recvcounts, rdispls, dtypes, ios->union_comm, + &iodesc->rearr_opts.io2comp))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + +#if PIO_ENABLE_LOGGING + for (int d = 0; d < ndims; d++) + LOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d])); +#endif /* PIO_ENABLE_LOGGING */ + + /* For each element of the data array on the compute task, + * find the IO task to send the data element to, and its + * offset into the global data array. */ + for (int k = 0; k < maplen; k++) { PIO_Offset gcoord[ndims], lcoord[ndims]; bool found = true; - /* The compmap array is 1 based but calculations are 0 based */ - idx_to_dim_list(ndims, gsize, compmap[k] - 1, gcoord); - for (j = 0; j < ndims; j++) + /* The compmap array is 1 based but calculations are 0 based */ + LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims)); + idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord); +#if PIO_ENABLE_LOGGING + for (int d = 0; d < ndims; d++) + LOG((3, "gcoord[%d] = %lld", d, gcoord[d])); +#endif /* PIO_ENABLE_LOGGING */ + + /* Find a destination for each entry in the compmap. */ + for (int j = 0; j < ndims; j++) { if (gcoord[j] >= start[j] && gcoord[j] < start[j] + count[j]) { @@ -1248,44 +1336,56 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com break; } } + + /* Did we find a destination IO task for this element + * of the computation task data array? If so, remember + * the destination IO task, and determine the index + * for that element in the IO task data. */ if (found) { dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count); dest_ioproc[k] = i; + LOG((3, "found dest_ioindex[%d] = %d dest_ioproc[%d] = %d", k, dest_ioindex[k], + k, dest_ioproc[k])); } } } } /* Check that a destination is found for each compmap entry. */ - for (k = 0; k < maplen; k++) + for (int k = 0; k < maplen; k++) if (dest_ioproc[k] < 0 && compmap[k] > 0) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - compute_counts(ios, iodesc, maplen, dest_ioproc, dest_ioindex, ios->union_comm); + /* Completes the mapping for the box rearranger. */ + LOG((2, "calling compute_counts maplen = %d", maplen)); + if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + /* Compute the max io buffer size needed for an iodesc. */ if (ios->ioproc) + { if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen)); + } - /* Using maxiobuflen compute the maximum number of vars of this type that the io - task buffer can handle. */ + /* Using maxiobuflen compute the maximum number of bytes that the + * io task buffer can handle. */ if ((ret = compute_maxaggregate_bytes(ios, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes)); -#ifdef DEBUG - iodesc_dump(iodesc); -#endif return PIO_NOERR; } /** - * Compare offsets is used by the sort in the subset rearrange. This + * Compare offsets is used by the sort in the subset rearranger. This * function is passed to qsort. * - * @param a - * @param b - * @returns 0 if offsets are the same. + * @param a pointer to an offset. + * @param b pointer to another offset. + * @returns 0 if offsets are the same or either pointer is NULL. */ int compare_offsets(const void *a, const void *b) { @@ -1297,60 +1397,75 @@ int compare_offsets(const void *a, const void *b) } /** + * Calculate start and count regions for the subset rearranger. This + * function is not used in the box rearranger. + * * Each region is a block of output which can be represented in a * single call to the underlying netcdf library. This can be as small * as a single data point, but we hope we've aggragated better than * that. * * @param ndims the number of dimensions - * @param gdims pointer to an array of dimension ids + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the length of the map - * @param map + * @param map may be NULL (when ???). * @param maxregions * @param firstregion pointer to the first region. * @returns 0 on success, error code otherwise. */ -void get_start_and_count_regions(int ndims, const int *gdims, int maplen, const PIO_Offset *map, - int *maxregions, io_region *firstregion) +int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map, + int *maxregions, io_region *firstregion) { - int i; - int nmaplen; + int nmaplen = 0; int regionlen; io_region *region; + int ret; - assert(maxregions); - assert(firstregion); + /* Check inputs. */ + pioassert(ndims >= 0 && gdimlen && maplen >= 0 && maxregions && firstregion, + "invalid input", __FILE__, __LINE__); + LOG((1, "get_regions ndims = %d maplen = %d", ndims, maplen)); - nmaplen = 0; region = firstregion; if (map) { - while(map[nmaplen++] <= 0); + while (map[nmaplen++] <= 0) + { + LOG((3, "map[%d] = %d", nmaplen, map[nmaplen])); + ; + } nmaplen--; } region->loffset = nmaplen; + LOG((2, "region->loffset = %d", region->loffset)); *maxregions = 1; - while(nmaplen < maplen) + while (nmaplen < maplen) { /* Here we find the largest region from the current offset - into the iomap regionlen is the size of that region and we + into the iomap. regionlen is the size of that region and we step to that point in the map array until we reach the - end */ - for (i = 0; i < ndims; i++) + end. */ + for (int i = 0; i < ndims; i++) region->count[i] = 1; - regionlen = find_region(ndims, gdims, maplen-nmaplen, - map+nmaplen, region->start, region->count); - + /* Set start/count to describe first region in map. */ + regionlen = find_region(ndims, gdimlen, maplen-nmaplen, + &map[nmaplen], region->start, region->count); pioassert(region->start[0] >= 0, "failed to find region", __FILE__, __LINE__); nmaplen = nmaplen + regionlen; + LOG((2, "regionlen = %d nmaplen = %d", regionlen, nmaplen)); + /* If we need to, allocate the next region. */ if (region->next == NULL && nmaplen < maplen) { - region->next = alloc_region(ndims); + LOG((2, "allocating next region")); + if ((ret = alloc_region2(NULL, ndims, ®ion->next))) + return ret; + /* The offset into the local array buffer is the sum of * the sizes of all of the previous regions (loffset) */ region = region->next; @@ -1361,28 +1476,39 @@ void get_start_and_count_regions(int ndims, const int *gdims, int maplen, const maxregions will be the total number of regions on this task. */ (*maxregions)++; + LOG((2, "*maxregions = %d", *maxregions)); } } + + return PIO_NOERR; } /** + * Create the MPI communicators needed by the subset rearranger. + * * The subset rearranger needs a mapping from compute tasks to IO * task, the only requirement is that each compute task map to one and * only one IO task. This mapping groups by mpi task id others are * possible and may be better for certain decompositions * - * @param ios pointer to the iosystem_desc_t struct + * The as yet unrealized vision here is that the user would be able to + * supply an alternative subset partitioning function. Requirements of + * this function are that there be exactly one io task per compute + * task group. + * + * @param ios pointer to the iosystem_desc_t struct. * @param iodesc a pointer to the io_desc_t struct. * @returns 0 on success, error code otherwise. */ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) { - int taskratio = ios->num_comptasks/ios->num_iotasks; int color; int key; int mpierr; /* Return value from MPI functions. */ - assert(ios && iodesc); + pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + LOG((1, "default_subset_partition ios->ioproc = %d ios->io_rank = %d " + "ios->comp_rank = %d", ios->ioproc, ios->io_rank, ios->comp_rank)); /* Create a new comm for each subset group with the io task in rank 0 and only 1 io task per group */ @@ -1393,10 +1519,13 @@ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) } else { + int taskratio = ios->num_comptasks / ios->num_iotasks; key = max(1, ios->comp_rank % taskratio + 1); color = min(ios->num_iotasks - 1, ios->comp_rank / taskratio); } + LOG((3, "key = %d color = %d", key, color)); + /* Create new communicators. */ if ((mpierr = MPI_Comm_split(ios->comp_comm, color, key, &iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -1404,34 +1533,72 @@ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc) } /** + * Create the subset rearranger. + * * The subset rearranger computes a mapping between IO tasks and * compute tasks such that each compute task communicates with one and * only one IO task. * + * This function is called from PIOc_InitDecomp(). + * + * This function: + *
    + *
  • Calls default_subset_partition() to create subset_comm. + *
  • For IO tasks, allocates iodesc->rcount array (length ntasks). + *
  • Allocates iodesc->scount array (length 1) + *
  • Determins value of iodesc->scount[0], the number of data + * elements on this compute task which are read/written. + *
  • Allocated and inits iodesc->sindex (length iodesc->scount[0]), + * init it to contain indicies to data. + *
  • Pass the reduced maplen (without holes) from each compute task + * to its associated IO task. + *
  • On IO tasks, determine llen. + *
  • Determine whether fill values will be needed. + *
  • Pass iodesc->sindex from each compute task to its associated IO + * task. + *
  • Create shrtmap, which is compmap without the holes. + *
  • Gather shrtmaps from each task into iomap. + *
  • On IO tasks, sort the mapping, this will transpose the data + * into IO order. + *
  • On IO tasks, allocate and init iodesc->rindex and iodesc->rfrom + * (length iodesc->llen). + *
  • On IO tasks, handle fill values, if needed. + *
  • On IO tasks, scatter values of srcindex to subset communicator. + *
  • On IO tasks, call get_regions() and distribute the max + * maxregions to all tasks in IO communicator. + *
  • On IO tasks, call compute_maxIObuffersize(). + *
  • Call compute_maxaggregate_bytes(). + *
+ * * @param ios pointer to the iosystem_desc_t struct. * @param maplen the length of the map. - * @param compmap - * @param gsize pointer to an array of sizes. + * @param compmap a 1 based array of offsets into the array record on + * file. A 0 in this array indicates a value which should not be + * transfered. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param ndims the number of dimensions. * @param iodesc a pointer to the io_desc_t struct. * @returns 0 on success, error code otherwise. */ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, - const int *gsize, int ndims, io_desc_t *iodesc) + const int *gdimlen, int ndims, io_desc_t *iodesc) { int i, j; PIO_Offset *iomap = NULL; - int ierr = PIO_NOERR; mapsort *map = NULL; PIO_Offset totalgridsize; PIO_Offset *srcindex = NULL; PIO_Offset *myfillgrid = NULL; int maxregions; - int rank, ntasks, rcnt; + int rank, ntasks; + int rcnt = 0; int mpierr; /* Return call from MPI function calls. */ int ret; - pioassert(iodesc, "iodesc must be provided", __FILE__, __LINE__); + /* Check inputs. */ + pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims >= 0 && iodesc, + "invalid input", __FILE__, __LINE__); LOG((2, "subset_rearrange_create maplen = %d ndims = %d", maplen, ndims)); @@ -1442,7 +1609,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma return pio_err(ios, NULL, ret, __FILE__, __LINE__); iodesc->rearranger = PIO_REARR_SUBSET; - /* Get size of this subset communicator, and rank in it. */ + /* Get size of this subset communicator and rank of this task in it. */ if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks))) @@ -1455,8 +1622,9 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma pioassert(rank > 0 && rank < ntasks, "Bad comp rank in subset create", __FILE__, __LINE__); - rcnt = 0; + /* Remember the maplen for this computation task. */ iodesc->ndof = maplen; + if (ios->ioproc) { /* Allocate space to hold count of data to be received in pio_swapm(). */ @@ -1471,10 +1639,15 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); iodesc->scount[0] = 0; + + /* Find the total size of the global data array. */ totalgridsize = 1; for (i = 0; i < ndims; i++) - totalgridsize *= gsize[i]; + totalgridsize *= gdimlen[i]; + /* Determine scount[0], the number of data elements in the + * computation task that are to be written, by looking at + * compmap. */ for (i = 0; i < maplen; i++) { /* turns out this can be allowed in some cases @@ -1484,6 +1657,8 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma (iodesc->scount[0])++; } + /* Allocate an array for indicies on the computation tasks (the + * send side when writing). */ if (iodesc->scount[0] > 0) if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -1493,24 +1668,25 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (compmap[i] > 0) iodesc->sindex[j++] = i; - /* Pass the reduced maplen (without holes) from each compute task to its associated IO task - printf("%s %d %ld\n",__FILE__,__LINE__,iodesc->scount); */ - if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, MPI_INT, - 0, iodesc->subset_comm))) + /* Pass the reduced maplen (without holes) from each compute task + * to its associated IO task. */ + if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt, + MPI_INT, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->llen = 0; int rdispls[ntasks]; - int recvlths[ntasks]; + int recvcounts[ntasks]; + /* On IO tasks determine llen. */ if (ios->ioproc) { - for (i = 0;i < ntasks; i++) + for (i = 0; i < ntasks; i++) { iodesc->llen += iodesc->rcount[i]; rdispls[i] = 0; - recvlths[i] = iodesc->rcount[i]; + recvcounts[i] = iodesc->rcount[i]; if (i > 0) rdispls[i] = rdispls[i - 1] + iodesc->rcount[i - 1]; } @@ -1528,20 +1704,24 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma { for (i = 0; i < ntasks; i++) { - recvlths[i] = 0; + recvcounts[i] = 0; rdispls[i] = 0; } } - if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + + /* Determine whether fill values will be needed. */ + if ((ret = determine_fill(ios, iodesc, gdimlen, compmap))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Pass the sindex from each compute task to its associated IO task. */ if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET, - srcindex, recvlths, rdispls, PIO_OFFSET, 0, + srcindex, recvcounts, rdispls, PIO_OFFSET, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (ios->ioproc && iodesc->llen>0) + /* On IO tasks which need it, allocate memory for the map and the + * iomap. */ + if (ios->ioproc && iodesc->llen > 0) { if (!(map = calloc(iodesc->llen, sizeof(mapsort)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -1552,13 +1732,13 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma /* Now pass the compmap, skipping the holes. */ PIO_Offset *shrtmap; - if (maplen>iodesc->scount[0] && iodesc->scount[0] > 0) + if (maplen > iodesc->scount[0] && iodesc->scount[0] > 0) { if (!(shrtmap = calloc(iodesc->scount[0], sizeof(PIO_Offset)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); j = 0; - for (i = 0; i < maplen; i++) + for (int i = 0; i < maplen; i++) if (compmap[i] > 0) shrtmap[j++] = compmap[i]; } @@ -1567,13 +1747,16 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma shrtmap = compmap; } - if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvlths, rdispls, - PIO_OFFSET, 0, iodesc->subset_comm))) + /* Gather shrtmap from each task in the subset communicator, and + * put gathered results into iomap. */ + if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvcounts, + rdispls, PIO_OFFSET, 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); if (shrtmap != compmap) free(shrtmap); + /* On IO tasks that have data in the local array ??? */ if (ios->ioproc && iodesc->llen > 0) { int pos = 0; @@ -1583,7 +1766,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma { for (j = 0; j < iodesc->rcount[i]; j++) { - mptr = map + k; + mptr = &map[k]; mptr->rfrom = i; mptr->soffset = srcindex[pos + j]; mptr->iomap = iomap[pos + j]; @@ -1591,6 +1774,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma } pos += iodesc->rcount[i]; } + /* sort the mapping, this will transpose the data into IO order */ qsort(map, iodesc->llen, sizeof(mapsort), compare_offsets); @@ -1605,21 +1789,20 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma for (i = 0; i < ntasks; i++) { cnt[i] = rdispls[i]; - - /* offsets to swapm are in bytes */ - /* rdispls[i]*=pio_offset_size; */ } - mapsort *mptr; + /* For IO tasks init rfrom and rindex arrays (compute tasks have + * llen of 0). */ for (i = 0; i < iodesc->llen; i++) { - mptr = map+i; + mapsort *mptr = &map[i]; iodesc->rfrom[i] = mptr->rfrom; iodesc->rindex[i] = i; iomap[i] = mptr->iomap; srcindex[(cnt[iodesc->rfrom[i]])++] = mptr->soffset; } + /* Handle fill values if needed. */ if (ios->ioproc && iodesc->needsfill) { /* we need the list of offsets which are not in the union of iomap */ @@ -1657,6 +1840,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma } } + /* Gather cnt from all tasks in the IO communicator into array gcnt. */ if ((mpierr = MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -1675,11 +1859,12 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma myusegrid[i] = -1; } - if ((mpierr = MPI_Gatherv((iomap + imin), cnt, PIO_OFFSET, myusegrid, gcnt, + if ((mpierr = MPI_Gatherv(&iomap[imin], cnt, PIO_OFFSET, myusegrid, gcnt, displs, PIO_OFFSET, nio, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } + /* Allocate and initialize a grid to fill in missing values. ??? */ PIO_Offset grid[thisgridsize[ios->io_rank]]; for (i = 0; i < thisgridsize[ios->io_rank]; i++) grid[i] = 0; @@ -1688,7 +1873,8 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma for (i = 0; i < thisgridsize[ios->io_rank]; i++) { int j = myusegrid[i] - thisgridmin[ios->io_rank]; - pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index", __FILE__, __LINE__); + pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index", + __FILE__, __LINE__); if (j >= 0) { grid[j] = 1; @@ -1698,7 +1884,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (myusegrid) free(myusegrid); - iodesc->holegridsize=thisgridsize[ios->io_rank] - cnt; + iodesc->holegridsize = thisgridsize[ios->io_rank] - cnt; if (iodesc->holegridsize > 0) { /* Allocate space for the fillgrid. */ @@ -1716,7 +1902,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (grid[i] == 0) { if (myfillgrid[j] == -1) - myfillgrid[j++]=thisgridmin[ios->io_rank] + i; + myfillgrid[j++] = thisgridmin[ios->io_rank] + i; else return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); } @@ -1725,34 +1911,52 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma iodesc->maxfillregions = 0; if (myfillgrid) { - iodesc->fillregion = alloc_region(iodesc->ndims); - get_start_and_count_regions(iodesc->ndims, gsize, iodesc->holegridsize, myfillgrid, - &iodesc->maxfillregions, iodesc->fillregion); + /* Allocate a data region to hold fill values. */ + if ((ret = alloc_region2(ios, iodesc->ndims, &iodesc->fillregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->holegridsize, myfillgrid, + &iodesc->maxfillregions, iodesc->fillregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); free(myfillgrid); maxregions = iodesc->maxfillregions; } - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); + /* Get the max maxregions, and distribute it to all tasks in + * the IO communicator. */ + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, + ios->io_comm))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->maxfillregions = maxregions; + /* Get the max maxholegridsize, and distribute it to all tasks + * in the IO communicator. */ iodesc->maxholegridsize = iodesc->holegridsize; - if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT, MPI_MAX, ios->io_comm))) + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT, + MPI_MAX, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } - if ((mpierr = MPI_Scatterv((void *)srcindex, recvlths, rdispls, PIO_OFFSET, (void *)iodesc->sindex, - iodesc->scount[0], PIO_OFFSET, 0, iodesc->subset_comm))) + /* Scatter values of srcindex to subset communicator. ??? */ + if ((mpierr = MPI_Scatterv((void *)srcindex, recvcounts, rdispls, PIO_OFFSET, + (void *)iodesc->sindex, iodesc->scount[0], PIO_OFFSET, + 0, iodesc->subset_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); + if (ios->ioproc) { iodesc->maxregions = 0; - get_start_and_count_regions(iodesc->ndims,gsize,iodesc->llen, iomap,&(iodesc->maxregions), - iodesc->firstregion); + if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->llen, iomap, + &iodesc->maxregions, iodesc->firstregion))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); maxregions = iodesc->maxregions; + + /* Get the max maxregions, and distribute it to all tasks in + * the IO communicator. */ if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); iodesc->maxregions = maxregions; + + /* Free resources. */ if (iomap) free(iomap); @@ -1762,12 +1966,11 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if (srcindex) free(srcindex); - compute_maxIObuffersize(ios->io_comm, iodesc); + /* Compute the max io buffer size needed for an iodesc. */ + if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); iodesc->nrecvs = ntasks; -#ifdef DEBUG - iodesc_dump(iodesc); -#endif } /* Using maxiobuflen compute the maximum number of vars of this type that the io @@ -1775,7 +1978,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma if ((ret = compute_maxaggregate_bytes(ios, iodesc))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - return ierr; + return PIO_NOERR; } /** @@ -1803,11 +2006,11 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) ibuf = NULL; if (iodesc->ndof > 0) if (!(cbuf = bget(iodesc->ndof * tsize))) - piomemerror(ios, iodesc->ndof * tsize, __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (iodesc->llen > 0) if (!(ibuf = bget(iodesc->llen * tsize))) - piomemerror(ios, iodesc->llen * tsize, __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (iodesc->rearranger == PIO_REARR_BOX) mycomm = ios->union_comm; @@ -1821,7 +2024,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) int log2 = log(nprocs) / log(2) + 1; if (!(wall = bget(2 * 4 * log2 * sizeof(double)))) - piomemerror(ios, 2 * 4 *log2 * sizeof(double), __FILE__, __LINE__); + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); double mintime; int k = 0; @@ -1835,16 +2038,16 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs; + handshake = iodesc->rearr_opts.comp2io.hs; isend = iodesc->isend; maxreqs = iodesc->max_requests; for (int i = 0; i < 2; i++) { if (i == 0) - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = false; + iodesc->rearr_opts.comp2io.hs = false; else - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = true; + iodesc->rearr_opts.comp2io.hs = true; for (int j = 0; j < 2; j++) { @@ -1871,7 +2074,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) if (wall[1] < mintime * 0.95) { - handshake = iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs; + handshake = iodesc->rearr_opts.comp2io.hs; isend = iodesc->isend; maxreqs = nreqs; mintime = wall[1]; @@ -1884,7 +2087,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) } } - iodesc->rearr_opts.comm_fc_opts_comp2io.enable_hs = handshake; + iodesc->rearr_opts.comp2io.hs = handshake; iodesc->isend = isend; iodesc->max_requests = maxreqs; diff --git a/src/externals/pio2/src/clib/pio_spmd.c b/src/externals/pio2/src/clib/pio_spmd.c index decf3f7e3bb1..c42fd60eff90 100644 --- a/src/externals/pio2/src/clib/pio_spmd.c +++ b/src/externals/pio2/src/clib/pio_spmd.c @@ -67,19 +67,12 @@ int pair(int np, int p, int k) * @param recvtypes array of datatypes (of length ntasks). Entry i * specifies the type of data received from process i. * @param comm MPI communicator for the MPI_Alltoallw call. - * @param handshake if true, use handshaking. - * @param isend the isend bool indicates whether sends should be - * posted using mpi_irsend which can be faster than blocking - * sends. When flow control is used max_requests > 0 and the number of - * irecvs posted from a given task will not exceed this value. On some - * networks too many outstanding irecvs will cause a communications - * bottleneck. - * @param max_requests If 0, no flow control is used. + * @param fc pointer to the struct that provided flow control options. * @returns 0 for success, error code otherwise. */ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes, void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes, - MPI_Comm comm, bool handshake, bool isend, int max_requests) + MPI_Comm comm, rearr_comm_fc_opt_t *fc) { int ntasks; /* Number of tasks in communicator comm. */ int my_rank; /* Rank of this task in comm. */ @@ -96,8 +89,8 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty MPI_Status status; /* Not actually used - replace with MPI_STATUSES_IGNORE. */ int mpierr; /* Return code from MPI functions. */ - LOG((2, "pio_swapm handshake = %d isend = %d max_requests = %d", handshake, - isend, max_requests)); + LOG((2, "pio_swapm fc->hs = %d fc->isend = %d fc->max_pend_req = %d", fc->hs, + fc->isend, fc->max_pend_req)); /* Get my rank and size of communicator. */ if ((mpierr = MPI_Comm_size(comm, &ntasks))) @@ -117,23 +110,15 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty #if PIO_ENABLE_LOGGING { for (int p = 0; p < ntasks; p++) - LOG((3, "sendcounts[%d] = %d", p, sendcounts[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "sdispls[%d] = %d", p, sdispls[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "sendtypes[%d] = %d", p, sendtypes[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "recvcounts[%d] = %d", p, recvcounts[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "rdispls[%d] = %d", p, rdispls[p])); - for (int p = 0; p < ntasks; p++) - LOG((3, "recvtypes[%d] = %d", p, recvtypes[p])); + LOG((3, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d " + "rdispls[%d] = %d recvtypes[%d] = %d", p, sendcounts[p], p, sdispls[p], p, + sendtypes[p], p, recvcounts[p], p, rdispls[p], p, recvtypes[p])); } #endif /* PIO_ENABLE_LOGGING */ - /* If max_requests == 0 no throttling is requested and the default + /* If fc->max_pend_req == 0 no throttling is requested and the default * mpi_alltoallw function is used. */ - if (max_requests == 0) + if (fc->max_pend_req == 0) { /* Call the MPI alltoall without flow control. */ LOG((3, "Calling MPI_Alltoallw without flow control.")); @@ -194,11 +179,11 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty swapids[i] = 0; } - if (isend) + if (fc->isend) for (int i = 0; i < ntasks; i++) sndids[i] = MPI_REQUEST_NULL; - if (handshake) + if (fc->hs) for (int i = 0; i < ntasks; i++) hs_rcvids[i] = MPI_REQUEST_NULL; @@ -222,17 +207,17 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty } else { - if (max_requests == PIO_REARR_COMM_UNLIMITED_PEND_REQ) + if (fc->max_pend_req == PIO_REARR_COMM_UNLIMITED_PEND_REQ) { maxreq = steps; maxreqh = steps; } - else if (max_requests > 1 && max_requests < steps) + else if (fc->max_pend_req > 1 && fc->max_pend_req < steps) { - maxreq = max_requests; + maxreq = fc->max_pend_req; maxreqh = maxreq / 2; } - else if (max_requests == 1) + else if (fc->max_pend_req == 1) { /* Note that steps >= 2 here */ maxreq = 2; @@ -245,11 +230,11 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty } } - LOG((2, "max_requests=%d, maxreq=%d, maxreqh=%d", max_requests, maxreq, maxreqh)); + LOG((2, "fc->max_pend_req=%d, maxreq=%d, maxreqh=%d", fc->max_pend_req, maxreq, maxreqh)); /* If handshaking is in use, do a nonblocking recieve to listen * for it. */ - if (handshake) + if (fc->hs) { for (istep = 0; istep < maxreq; istep++) { @@ -276,7 +261,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty rcvids + istep))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (handshake) + if (fc->hs) if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } @@ -292,7 +277,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty tag = my_rank + offset_t; /* If handshake is enabled don't post sends until the * receiving task has posted recvs. */ - if (handshake) + if (fc->hs) { if ((mpierr = MPI_Wait(hs_rcvids + istep, &status))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); @@ -308,7 +293,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty * used to choose between mpi_irsends and mpi_isends - the default * is still mpi_irsend */ - if (handshake && isend) + if (fc->hs && fc->isend) { #ifdef USE_MPI_ISEND_FOR_FC if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm, @@ -320,7 +305,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty return check_mpi(NULL, mpierr, __FILE__, __LINE__); #endif } - else if (isend) + else if (fc->isend) { if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm, sndids + istep))) @@ -347,7 +332,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty if (rstep < steps) { p = swapids[rstep]; - if (handshake && sendcounts[p] > 0) + if (fc->hs && sendcounts[p] > 0) { tag = my_rank + offset_t; if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep))) @@ -360,7 +345,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty ptr = (char *)recvbuf + rdispls[p]; if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm, rcvids + rstep))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (handshake) + if (fc->hs) if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } @@ -376,7 +361,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty LOG((2, "Waiting for outstanding msgs")); if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (isend) + if (fc->isend) if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); } diff --git a/src/externals/pio2/src/clib/pio_varm.c b/src/externals/pio2/src/clib/pio_varm.c index 18d55c5aea88..f02cb196c911 100644 --- a/src/externals/pio2/src/clib/pio_varm.c +++ b/src/externals/pio2/src/clib/pio_varm.c @@ -29,7 +29,7 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -98,7 +98,7 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -167,7 +167,7 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -235,7 +235,7 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -306,7 +306,7 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -377,7 +377,7 @@ int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -447,7 +447,7 @@ int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -518,7 +518,7 @@ int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -588,7 +588,7 @@ int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -660,7 +660,7 @@ int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -731,7 +731,7 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -801,7 +801,7 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; @@ -872,7 +872,7 @@ int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ios = file->iosystem; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -941,7 +941,7 @@ int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -979,7 +979,7 @@ int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1010,7 +1010,7 @@ int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1048,7 +1048,7 @@ int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1079,7 +1079,7 @@ int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const P ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1117,7 +1117,7 @@ int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const P ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1148,7 +1148,7 @@ int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1186,7 +1186,7 @@ int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1217,7 +1217,7 @@ int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1255,7 +1255,7 @@ int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1286,7 +1286,7 @@ int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1324,7 +1324,7 @@ int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1350,7 +1350,7 @@ int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1388,7 +1388,7 @@ int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1419,7 +1419,7 @@ int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1457,7 +1457,7 @@ int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1488,7 +1488,7 @@ int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1526,7 +1526,7 @@ int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1557,7 +1557,7 @@ int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1595,7 +1595,7 @@ int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1626,7 +1626,7 @@ int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1664,7 +1664,7 @@ int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1695,7 +1695,7 @@ int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1733,7 +1733,7 @@ int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -1764,7 +1764,7 @@ int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ierr = PIO_NOERR; /* Sorry, but varm functions are not supported by the async interface. */ - if (ios->async_interface) + if (ios->async) return PIO_EINVAL; if (ios->ioproc){ @@ -1802,7 +1802,7 @@ int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if (ios->async_interface || bcast || + if (ios->async || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } diff --git a/src/externals/pio2/src/clib/pioc.c b/src/externals/pio2/src/clib/pioc.c index 9612f65c8eed..0dc4fed89fd6 100644 --- a/src/externals/pio2/src/clib/pioc.c +++ b/src/externals/pio2/src/clib/pioc.c @@ -208,7 +208,7 @@ int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method) /* Get the iosystem info. */ if (iosysid != PIO_DEFAULT) if (!(ios = pio_get_iosystem_from_id(iosysid))) - return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + piodie("Could not find IO system.", __FILE__, __LINE__); /* Set the error handler. */ if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod)) @@ -249,7 +249,7 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) /* If using async, and not an IO task, then send parameters. */ if (iosysid != PIO_DEFAULT) - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -290,11 +290,25 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) * decomposition describes how the data will be distributed between * tasks. * + * Internally, this function will: + *
    + *
  • Allocate and initialize an iodesc struct for this + * decomposition. (This also allocates an io_region struct for the + * first region.) + *
  • (Box rearranger only) If iostart or iocount are NULL, call + * CalcStartandCount() to determine starts/counts. Then call + * compute_maxIObuffersize() to compute the max IO buffer size needed. + *
  • Create the rearranger. + *
  • Assign an ioid and add this decomposition to the list of open + * decompositions. + *
+ * * @param iosysid the IO system ID. - * @param basetype the basic PIO data type used. + * @param pio_type the basic PIO data type used. * @param ndims the number of dimensions in the variable, not * including the unlimited dimension. - * @param dims an array of global size of each dimension. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the local length of the compmap array. * @param compmap a 1 based array of offsets into the array record on * file. A 0 in this array indicates a value which should not be @@ -303,13 +317,17 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) * @param rearranger pointer to the rearranger to be used for this * decomp or NULL to use the default. * @param iostart An array of start values for block cyclic - * decompositions. If NULL ??? + * decompositions for the SUBSET rearranger. Ignored if block + * rearranger is used. If NULL and SUBSET rearranger is used, the + * iostarts are generated. * @param iocount An array of count values for block cyclic - * decompositions. If NULL ??? + * decompositions for the SUBSET rearranger. Ignored if block + * rearranger is used. If NULL and SUBSET rearranger is used, the + * iostarts are generated. * @returns 0 on success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, +int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount) { @@ -318,24 +336,24 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */ int ierr; /* Return code. */ - LOG((1, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d", - iosysid, basetype, ndims, maplen)); + LOG((1, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d", + iosysid, pio_type, ndims, maplen)); /* Get IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Caller must provide these. */ - if (!dims || !compmap || !ioidp) + if (!gdimlen || !compmap || !ioidp) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Check the dim lengths. */ for (int i = 0; i < ndims; i++) - if (dims[i] <= 0) + if (gdimlen[i] <= 0) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -350,11 +368,11 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m if (!mpierr) mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&basetype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((int *)dims, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -374,8 +392,8 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (iocount_present && !mpierr) mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_InitDecomp iosysid = %d basetype = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " - "iocount_present = %d ", iosysid, basetype, ndims, maplen, rearranger_present, iostart_present, iocount_present)); + LOG((2, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " + "iocount_present = %d ", iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); } /* Handle MPI errors. */ @@ -385,8 +403,10 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); } - /* Allocate space for the iodesc info. */ - if ((ierr = malloc_iodesc(ios, basetype, ndims, &iodesc))) + /* Allocate space for the iodesc info. This also allocates the + * first region and copies the rearranger opts into this + * iodesc. */ + if ((ierr = malloc_iodesc(ios, pio_type, ndims, &iodesc))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); /* Remember the maplen. */ @@ -402,7 +422,7 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m if (!(iodesc->dimlen = malloc(sizeof(int) * ndims))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int d = 0; d < ndims; d++) - iodesc->dimlen[d] = dims[d]; + iodesc->dimlen[d] = gdimlen[d]; /* Set the rearranger. */ if (!rearranger) @@ -414,71 +434,76 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m /* Is this the subset rearranger? */ if (iodesc->rearranger == PIO_REARR_SUBSET) { - LOG((2, "Handling subset rearranger.")); - if (iostart && iocount) - fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp", - "are incompatable with subset rearrange method and will be ignored"); iodesc->num_aiotasks = ios->num_iotasks; - if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, dims, + LOG((2, "creating subset rearranger iodesc->num_aiotasks = %d", + iodesc->num_aiotasks)); + if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, gdimlen, ndims, iodesc))) - return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } - else + else /* box rearranger */ { - LOG((2, "Handling not the subset rearranger.")); if (ios->ioproc) { /* Unless the user specifies the start and count for each * IO task compute it. */ if (iostart && iocount) { - iodesc->maxiobuflen = 1; + LOG((3, "iostart and iocount provided")); for (int i = 0; i < ndims; i++) { iodesc->firstregion->start[i] = iostart[i]; iodesc->firstregion->count[i] = iocount[i]; - compute_maxIObuffersize(ios->io_comm, iodesc); - } iodesc->num_aiotasks = ios->num_iotasks; } else { - iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims, - ios->num_iotasks, ios->io_rank, - iodesc->firstregion->start, iodesc->firstregion->count); + /* Compute start and count values for each io task. */ + LOG((2, "about to call CalcStartandCount pio_type = %d ndims = %d", pio_type, ndims)); + if ((ierr = CalcStartandCount(pio_type, ndims, gdimlen, ios->num_iotasks, + ios->io_rank, iodesc->firstregion->start, + iodesc->firstregion->count, &iodesc->num_aiotasks))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } - compute_maxIObuffersize(ios->io_comm, iodesc); + + /* Compute the max io buffer size needed for an iodesc. */ + if ((ierr = compute_maxIObuffersize(ios->io_comm, iodesc))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); + LOG((3, "compute_maxIObuffersize called iodesc->maxiobuflen = %d", + iodesc->maxiobuflen)); } /* Depending on array size and io-blocksize the actual number * of io tasks used may vary. */ - if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, + ios->my_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((3, "iodesc->num_aiotasks = %d", iodesc->num_aiotasks)); /* Compute the communications pattern for this decomposition. */ if (iodesc->rearranger == PIO_REARR_BOX) - ierr = box_rearrange_create(ios, maplen, compmap, dims, ndims, iodesc); - - /* - if (ios->ioproc){ - io_region *ioregion = iodesc->firstregion; - while(ioregion != NULL){ - for (int i=0;istart[i],ioregion->count[i]); - ioregion = ioregion->next; - } - } - */ + if ((ierr = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } /* Add this IO description to the list. */ *ioidp = pio_add_to_iodesc_list(iodesc); - LOG((3, "About to tune rearranger...")); +#if PIO_ENABLE_LOGGING + /* Log results. */ + LOG((2, "iodesc ioid = %d nrecvs = %d ndof = %d ndims = %d num_aiotasks = %d " + "rearranger = %d maxregions = %d needsfill = %d llen = %d maxiobuflen = %d", + iodesc->ioid, iodesc->nrecvs, iodesc->ndof, iodesc->ndims, iodesc->num_aiotasks, + iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->llen, + iodesc->maxiobuflen)); + for (int j = 0; j < iodesc->llen; j++) + LOG((3, "rindex[%d] = %lld", j, iodesc->rindex[j])); +#endif /* PIO_ENABLE_LOGGING */ + + /* This function only does something if pre-processor macro + * PERFTUNE is set. */ performance_tune_rearranger(ios, iodesc); - LOG((3, "Done with rearranger tune.")); return PIO_NOERR; } @@ -489,17 +514,19 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m * tasks. * * @param iosysid the IO system ID. - * @param basetype the basic PIO data type used. + * @param pio_type the basic PIO data type used. * @param ndims the number of dimensions in the variable, not * including the unlimited dimension. - * @param dims an array of global size of each dimension. + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param maplen the local length of the compmap array. * @param compmap a 0 based array of offsets into the array record on * file. A -1 in this array indicates a value which should not be * transfered. * @param ioidp pointer that will get the io description ID. - * @param rearranger pointer to the rearranger to be used for this - * decomp or NULL to use the default. + * @param rearranger the rearranger to be used for this decomp or 0 to + * use the default. Valid rearrangers are PIO_REARR_BOX and + * PIO_REARR_SUBSET. * @param iostart An array of start values for block cyclic * decompositions. If NULL ??? * @param iocount An array of count values for block cyclic @@ -507,21 +534,30 @@ int PIOc_InitDecomp(int iosysid, int basetype, int ndims, const int *dims, int m * @returns 0 on success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int maplen, - const PIO_Offset *compmap, int *ioidp, const int *rearranger, +int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen, + const PIO_Offset *compmap, int *ioidp, int rearranger, const PIO_Offset *iostart, const PIO_Offset *iocount) { PIO_Offset compmap_1_based[maplen]; + int *rearrangerp = NULL; + + LOG((1, "PIOc_init_decomp iosysid = %d pio_type = %d ndims = %d maplen = %d", + iosysid, pio_type, ndims, maplen)); - LOG((1, "PIOc_init_decomp iosysid = %d basetype = %d ndims = %d maplen = %d", - iosysid, basetype, ndims, maplen)); + /* If the user specified a non-default rearranger, use it. */ + if (rearranger) + rearrangerp = &rearranger; /* Add 1 to all elements in compmap. */ for (int e = 0; e < maplen; e++) + { + LOG((3, "zero-based compmap[%d] = %d", e, compmap[e])); compmap_1_based[e] = compmap[e] + 1; + } - return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap_1_based, - ioidp, rearranger, iostart, iocount); + /* Call the legacy version of the function. */ + return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based, + ioidp, rearrangerp, iostart, iocount); } /** @@ -530,16 +566,17 @@ int PIOc_init_decomp(int iosysid, int basetype, int ndims, const int *dims, int * the file. In this case we compute the compdof. * * @param iosysid the IO system ID - * @param basetype + * @param pio_type * @param ndims the number of dimensions - * @param dims array of dimensions + * @param gdimlen an array length ndims with the sizes of the global + * dimensions. * @param start start array * @param count count array * @param pointer that gets the IO ID. * @returns 0 for success, error code otherwise * @ingroup PIO_initdecomp */ -int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, +int PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen, const long int *start, const long int *count, int *ioidp) { @@ -548,20 +585,20 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, PIO_Offset prod[ndims], loc[ndims]; int rearr = PIO_REARR_SUBSET; - LOG((1, "PIOc_InitDecomp_bc iosysid = %d basetype = %d ndims = %d")); + LOG((1, "PIOc_InitDecomp_bc iosysid = %d pio_type = %d ndims = %d")); /* Get the info about the io system. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Check for required inputs. */ - if (!dims || !start || !count || !ioidp) + if (!gdimlen || !start || !count || !ioidp) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Check that dim, start, and count values are not obviously * incorrect. */ for (int i = 0; i < ndims; i++) - if (dims[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > dims[i]) + if (gdimlen[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > gdimlen[i]) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Find the maplen. */ @@ -576,7 +613,7 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, loc[ndims - 1] = 0; for (n = ndims - 2; n >= 0; n--) { - prod[n] = prod[n + 1] * dims[n + 1]; + prod[n] = prod[n + 1] * gdimlen[n + 1]; loc[n] = 0; } for (i = 0; i < maplen; i++) @@ -594,7 +631,7 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, } } - return PIOc_InitDecomp(iosysid, basetype, ndims, dims, maplen, compmap, ioidp, + return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap, ioidp, &rearr, NULL, NULL); } @@ -608,13 +645,37 @@ int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *dims, * The caller must create all comp_comm and the io_comm MPI * communicators before calling this function. * - * @param comp_comm the MPI_Comm of the compute tasks - * @param num_iotasks the number of io tasks to use - * @param stride the offset between io tasks in the comp_comm - * @param base the comp_comm index of the first io task + * Internally, this function does the following: + * + *
    + *
  • Initialize logging system (if PIO_ENABLE_LOGGING is set). + *
  • Allocates and initializes the iosystem_desc_t struct (ios). + *
  • MPI duplicated user comp_comm to ios->comp_comm and + * ios->union_comm. + *
  • Set ios->my_comm to be ios->comp_comm. (Not an MPI + * duplication.) + *
  • Find MPI rank in comp_comm, determine ranks of IO tasks, + * determine whether this task is one of the IO tasks. + *
  • Identify the root IO tasks. + *
  • Create MPI groups for IO tasks, and for computation tasks. + *
  • On IO tasks, create an IO communicator (ios->io_comm). + *
  • Assign an iosystemid, and put this iosystem_desc_t into the + * list of open iosystems. + *
  • Initialize the bget buffer, unless PIO_USE_MALLOC was used. + *
+ * + * When complete, there are three MPI communicators (ios->comp_comm, + * ios->union_comm, and ios->io_comm), and two MPI groups + * (ios->compgroup and ios->iogroup) that must be freed by MPI. + * + * @param comp_comm the MPI_Comm of the compute tasks. + * @param num_iotasks the number of io tasks to use. + * @param stride the offset between io tasks in the comp_comm. + * @param base the comp_comm index of the first io task. * @param rearr the rearranger to use by default, this may be - * overriden in the @ref PIO_initdecomp - * @param iosysidp index of the defined system descriptor + * overriden in the PIO_init_decomp(). The rearranger is not used + * until the decomposition is initialized. + * @param iosysidp index of the defined system descriptor. * @return 0 on success, otherwise a PIO error code. * @ingroup PIO_init */ @@ -652,9 +713,13 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->num_iotasks = num_iotasks; ios->num_comptasks = num_comptasks; - /* Initialize the rearranger options. */ - init_rearr_opts(ios); + /* For non-async, the IO tasks are a subset of the comptasks. */ + ios->num_uniontasks = num_comptasks; + /* Initialize the rearranger options. */ + ios->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + ios->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + /* Copy the computation communicator into union_comm. */ if ((mpierr = MPI_Comm_dup(comp_comm, &ios->union_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -667,35 +732,42 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->my_comm = ios->comp_comm; ustride = stride; - /* Find MPI rank comp_comm communicator. */ + /* Find MPI rank in comp_comm communicator. */ if ((mpierr = MPI_Comm_rank(ios->comp_comm, &ios->comp_rank))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + /* With non-async, all tasks are part of computation component. */ + ios->compproc = true; + + /* Create an array that holds the ranks of the tasks to be used + * for computation. */ + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < ios->num_comptasks; i++) + ios->compranks[i] = i; + + /* Is this the comp master? */ if (ios->comp_rank == 0) ios->compmaster = MPI_ROOT; LOG((2, "comp_rank = %d num_comptasks = %d", ios->comp_rank, ios->num_comptasks)); /* Create an array that holds the ranks of the tasks to be used - * for IO. NOTE that sizeof(int) should probably be 1, not - * sizeof(int) ???*/ - if (!(ios->ioranks = calloc(sizeof(int), ios->num_iotasks))) + * for IO. */ + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int i = 0; i < ios->num_iotasks; i++) { ios->ioranks[i] = (base + i * ustride) % ios->num_comptasks; if (ios->ioranks[i] == ios->comp_rank) ios->ioproc = true; + LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); } ios->ioroot = ios->ioranks[0]; - for (int i = 0; i < ios->num_iotasks; i++) - LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); - /* We are not providing an info object. */ ios->info = MPI_INFO_NULL; - /* The task that has an iomaster value of MPI_ROOT will be the - * root of the IO communicator. */ + /* Identify the task that will be the root of the IO communicator. */ if (ios->comp_rank == ios->ioranks[0]) ios->iomaster = MPI_ROOT; @@ -713,8 +785,8 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); /* For the tasks that are doing IO, get their rank within the IO - * communicator. For some reason when I check the return value of - * this MPI call, all tests start to fail! */ + * communicator. If they are not doing IO, set their io_rank to + * -1. */ if (ios->ioproc) { if ((mpierr = MPI_Comm_rank(ios->io_comm, &ios->io_rank))) @@ -724,6 +796,7 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas ios->io_rank = -1; LOG((3, "ios->io_comm = %d ios->io_rank = %d", ios->io_comm, ios->io_rank)); + /* Rank in the union comm is the same as rank in the comp comm. */ ios->union_rank = ios->comp_rank; /* Add this ios struct to the list in the PIO library. */ @@ -770,12 +843,12 @@ int PIOc_Init_Intracomm_from_F90(int f90_comp_comm, LOG((1, "Setting rearranger options, iosys=%d", *iosysidp)); return PIOc_set_rearr_opts(*iosysidp, rearr_opts->comm_type, rearr_opts->fcd, - rearr_opts->comm_fc_opts_comp2io.enable_hs, - rearr_opts->comm_fc_opts_comp2io.enable_isend, - rearr_opts->comm_fc_opts_comp2io.max_pend_req, - rearr_opts->comm_fc_opts_io2comp.enable_hs, - rearr_opts->comm_fc_opts_io2comp.enable_isend, - rearr_opts->comm_fc_opts_io2comp.max_pend_req); + rearr_opts->comp2io.hs, + rearr_opts->comp2io.isend, + rearr_opts->comp2io.max_pend_req, + rearr_opts->io2comp.hs, + rearr_opts->io2comp.isend, + rearr_opts->io2comp.max_pend_req); } return ret; } @@ -841,7 +914,7 @@ int PIOc_finalize(int iosysid) * comp master to the IO processes. This may be called by * componets for other components iosysid. So don't send unless * there is a valid union_comm. */ - if (ios->async_interface && ios->union_comm != MPI_COMM_NULL) + if (ios->async && ios->union_comm != MPI_COMM_NULL) { int msg = PIO_MSG_EXIT; @@ -874,6 +947,9 @@ int PIOc_finalize(int iosysid) if (ios->ioranks) free(ios->ioranks); LOG((3, "Freed ioranks.")); + if (ios->compranks) + free(ios->compranks); + LOG((3, "Freed compranks.")); /* Free the buffer pool. */ int niosysid; @@ -1062,24 +1138,30 @@ int PIOc_iotype_available(int iotype) * duplicates and each must later be freed with MPI_Free() by the * caller.) * + * @param rearranger the default rearranger to use for decompositions + * in this IO system. Must be either PIO_REARR_BOX or + * PIO_REARR_SUBSET. + * * @param iosysidp pointer to array of length component_count that * gets the iosysid for each component. * * @return PIO_NOERR on success, error code otherwise. * @ingroup PIO_init */ -int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, +int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count, int *num_procs_per_comp, int **proc_list, - MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int *iosysidp) + MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int rearranger, + int *iosysidp) { - int my_rank; - int **my_proc_list; - int *my_io_proc_list; - int mpierr; - int ret; + int my_rank; /* Rank of this task. */ + int **my_proc_list; /* Array of arrays of procs for comp components. */ + int *my_io_proc_list; /* List of processors in IO component. */ + int mpierr; /* Return code from MPI functions. */ + int ret; /* Return code. */ /* Check input parameters. */ - if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp) + if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp || + (rearranger != PIO_REARR_BOX && rearranger != PIO_REARR_SUBSET)) return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); /* Temporarily limit to one computational component. */ @@ -1088,16 +1170,21 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Turn on the logging system for PIO. */ pio_init_logging(); - LOG((1, "PIOc_Init_Async component_count = %d", component_count)); + LOG((1, "PIOc_Init_Async num_io_procs = %d component_count = %d", num_io_procs, + component_count)); /* If the user did not supply a list of process numbers to use for * IO, create it. */ if (!io_proc_list) { + LOG((3, "calculating processors for IO component")); if (!(my_io_proc_list = malloc(num_io_procs * sizeof(int)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int p = 0; p < num_io_procs; p++) + { my_io_proc_list[p] = p; + LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p])); + } } else my_io_proc_list = io_proc_list; @@ -1106,16 +1193,16 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, * component, create one. */ if (!proc_list) { - int last_proc = 0; + int last_proc = num_io_procs; /* Allocate space for array of arrays. */ - if (!(my_proc_list = malloc((component_count + 1) * sizeof(int *)))) + if (!(my_proc_list = malloc((component_count) * sizeof(int *)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Fill the array of arrays. */ - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) { - LOG((3, "calculating processors for component %d", cmp)); + LOG((3, "calculating processors for component %d num_procs_per_comp[cmp] = %d", cmp, num_procs_per_comp[cmp])); /* Allocate space for each array. */ if (!(my_proc_list[cmp] = malloc(num_procs_per_comp[cmp] * sizeof(int)))) @@ -1133,20 +1220,19 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, else my_proc_list = proc_list; - /* Get rank of this task. */ + /* Get rank of this task in world. */ if ((ret = MPI_Comm_rank(world, &my_rank))) return check_mpi(NULL, ret, __FILE__, __LINE__); /* Is this process in the IO component? */ int pidx; - for (pidx = 0; pidx < num_procs_per_comp[0]; pidx++) - if (my_rank == my_proc_list[0][pidx]) + for (pidx = 0; pidx < num_io_procs; pidx++) + if (my_rank == my_io_proc_list[pidx]) break; - int in_io = (pidx == num_procs_per_comp[0]) ? 0 : 1; + int in_io = (pidx == num_io_procs) ? 0 : 1; LOG((3, "in_io = %d", in_io)); /* Allocate struct to hold io system info for each computation component. */ - /* Allocate struct to hold io system info for each component. */ iosystem_desc_t *iosys[component_count], *my_iosys; for (int cmp1 = 0; cmp1 < component_count; cmp1++) if (!(iosys[cmp1] = (iosystem_desc_t *)calloc(1, sizeof(iosystem_desc_t)))) @@ -1174,9 +1260,7 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Create a group for the IO component. */ if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group))) return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "created IO group - io_group = %d group empty is %d", io_group, MPI_GROUP_EMPTY)); - for (int p = 0; p < num_io_procs; p++) - LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p])); + LOG((3, "created IO group - io_group = %d MPI_GROUP_EMPTY = %d", io_group, MPI_GROUP_EMPTY)); /* There is one shared IO comm. Create it. */ if ((ret = MPI_Comm_create(world, io_group, &io_comm))) @@ -1204,45 +1288,47 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); } - /* We will create a group for each component. */ - MPI_Group group[component_count + 1]; + /* We will create a group for each computational component. */ + MPI_Group group[component_count]; /* We will also create a group for each component and the IO * component processes (i.e. a union of computation and IO * processes. */ MPI_Group union_group[component_count]; - /* For each component, starting with the IO component. */ - for (int cmp = 0; cmp < component_count + 1; cmp++) + /* For each computation component. */ + for (int cmp = 0; cmp < component_count; cmp++) { LOG((3, "processing component %d", cmp)); - /* Don't start initing iosys until after IO component. */ - if (cmp) - { - /* Get pointer to current iosys. */ - my_iosys = iosys[cmp - 1]; - - /* Initialize some values. */ - my_iosys->io_comm = MPI_COMM_NULL; - my_iosys->comp_comm = MPI_COMM_NULL; - my_iosys->union_comm = MPI_COMM_NULL; - my_iosys->intercomm = MPI_COMM_NULL; - my_iosys->my_comm = MPI_COMM_NULL; - my_iosys->async_interface = 1; - my_iosys->error_handler = default_error_handler; - my_iosys->num_comptasks = num_procs_per_comp[cmp]; - my_iosys->num_iotasks = num_procs_per_comp[0]; - my_iosys->compgroup = MPI_GROUP_NULL; - my_iosys->iogroup = MPI_GROUP_NULL; - - /* The rank of the computation leader in the union comm. */ - my_iosys->comproot = num_procs_per_comp[0]; - LOG((3, "my_iosys->comproot = %d", my_iosys->comproot)); - - /* We are not providing an info object. */ - my_iosys->info = MPI_INFO_NULL; - } + /* Get pointer to current iosys. */ + my_iosys = iosys[cmp]; + + /* Initialize some values. */ + my_iosys->io_comm = MPI_COMM_NULL; + my_iosys->comp_comm = MPI_COMM_NULL; + my_iosys->union_comm = MPI_COMM_NULL; + my_iosys->intercomm = MPI_COMM_NULL; + my_iosys->my_comm = MPI_COMM_NULL; + my_iosys->async = 1; + my_iosys->error_handler = default_error_handler; + my_iosys->num_comptasks = num_procs_per_comp[cmp]; + my_iosys->num_iotasks = num_io_procs; + my_iosys->num_uniontasks = my_iosys->num_comptasks + my_iosys->num_iotasks; + my_iosys->compgroup = MPI_GROUP_NULL; + my_iosys->iogroup = MPI_GROUP_NULL; + my_iosys->default_rearranger = rearranger; + + /* Initialize the rearranger options. */ + my_iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + my_iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* The rank of the computation leader in the union comm. */ + my_iosys->comproot = num_io_procs; + LOG((3, "my_iosys->comproot = %d", my_iosys->comproot)); + + /* We are not providing an info object. */ + my_iosys->info = MPI_INFO_NULL; /* Create a group for this component. */ if ((ret = MPI_Group_incl(world_group, num_procs_per_comp[cmp], my_proc_list[cmp], @@ -1250,39 +1336,47 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, return check_mpi(NULL, ret, __FILE__, __LINE__); LOG((3, "created component MPI group - group[%d] = %d", cmp, group[cmp])); - /* For all the computation components (i.e. cmp != 0), create - * a union group with their processors and the processors of - * the (shared) IO component. */ - if (cmp) - { - /* How many processors in the union comm? */ - int nprocs_union = num_procs_per_comp[0] + num_procs_per_comp[cmp]; + /* For all the computation components create a union group + * with their processors and the processors of the (shared) IO + * component. */ - /* This will hold proc numbers from both computation and IO - * components. */ - int proc_list_union[nprocs_union]; + /* How many processors in the union comm? */ + int nprocs_union = num_io_procs + num_procs_per_comp[cmp]; - /* Add proc numbers from IO. */ - for (int p = 0; p < num_procs_per_comp[0]; p++) - proc_list_union[p] = my_proc_list[0][p]; + /* This will hold proc numbers from both computation and IO + * components. */ + int proc_list_union[nprocs_union]; - /* Add proc numbers from computation component. */ - for (int p = 0; p < num_procs_per_comp[cmp]; p++) - proc_list_union[p + num_procs_per_comp[0]] = my_proc_list[cmp][p]; + /* Add proc numbers from IO. */ + for (int p = 0; p < num_io_procs; p++) + proc_list_union[p] = my_io_proc_list[p]; - /* Create the union group. */ - if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, - &union_group[cmp - 1]))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp, union_group[cmp-1], nprocs_union)); - } + /* Add proc numbers from computation component. */ + for (int p = 0; p < num_procs_per_comp[cmp]; p++) + proc_list_union[p + num_io_procs] = my_proc_list[cmp][p]; + + /* Allocate space for computation task ranks. */ + if (!(my_iosys->compranks = calloc(my_iosys->num_comptasks, sizeof(int)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Remember computation task ranks. */ + for (int p = 0; p < num_procs_per_comp[cmp]; p++) + my_iosys->compranks[p] = my_proc_list[cmp][p]; + + /* Create the union group. */ + if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, &union_group[cmp]))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp, + union_group[cmp], nprocs_union)); /* Remember whether this process is in the IO component. */ - if (cmp) - my_iosys->ioproc = in_io; + my_iosys->ioproc = in_io; + + /* With async, tasks are either in a computation component or + * the IO component. */ + my_iosys->compproc = !in_io; - /* Is this process in this computation component (which is the - * IO component if cmp == 0)? */ + /* Is this process in this computation component? */ int in_cmp = 0; for (pidx = 0; pidx < num_procs_per_comp[cmp]; pidx++) if (my_rank == my_proc_list[cmp][pidx]) @@ -1294,113 +1388,97 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Create an intracomm for this component. Only processes in * the component need to participate in the intracomm create * call. */ - /* Create the intracomm from the group. */ LOG((3, "creating intracomm cmp = %d from group[%d] = %d", cmp, cmp, group[cmp])); + if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* We handle the IO comm differently (cmp == 0). */ - if (!cmp) + if (in_cmp) { - /* LOG((3, "about to create io comm")); */ - /* if ((ret = MPI_Comm_create_group(world, group[cmp], cmp, &io_comm))) */ - /* return check_mpi(NULL, ret, __FILE__, __LINE__); */ - /* LOG((3, "about to get io rank")); */ - /* if ((ret = MPI_Comm_rank(io_comm, &io_rank))) */ - /* return check_mpi(NULL, ret, __FILE__, __LINE__); */ - /* iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL; */ - /* LOG((3, "intracomm created for cmp = %d io_comm = %d io_rank = %d IO %s", */ - /* cmp, io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); */ - } - else - { - if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + /* Does the user want a copy? */ + if (user_comp_comm) + if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); - if (in_cmp) - { - /* Does the user want a copy? */ - if (user_comp_comm) - if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp - 1]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - - /* Get the rank in this comp comm. */ - if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + /* Get the rank in this comp comm. */ + if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* Set comp_rank 0 to be the compmaster. It will have - * a setting of MPI_ROOT, all other tasks will have a - * setting of MPI_PROC_NULL. */ - my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; + /* Set comp_rank 0 to be the compmaster. It will have a + * setting of MPI_ROOT, all other tasks will have a + * setting of MPI_PROC_NULL. */ + my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; - LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s", - cmp, my_iosys->comp_comm, my_iosys->comp_rank, - my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT")); - } + LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s", + cmp, my_iosys->comp_comm, my_iosys->comp_rank, + my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT")); } - /* If this is the IO component, make a copy of the IO comm for * each computational component. */ if (in_io) - if (cmp) - { - LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank)); - if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank)); - my_iosys->iomaster = iomaster; - my_iosys->io_rank = io_rank; - my_iosys->ioroot = 0; - my_iosys->comp_idx = cmp - 1; - } + { + LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank)); + if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank)); + my_iosys->iomaster = iomaster; + my_iosys->io_rank = io_rank; + my_iosys->ioroot = 0; + my_iosys->comp_idx = cmp; + } + + /* Create an array that holds the ranks of the tasks to be used + * for IO. */ + if (!(my_iosys->ioranks = calloc(my_iosys->num_iotasks, sizeof(int)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < my_iosys->num_iotasks; i++) + my_iosys->ioranks[i] = my_io_proc_list[i]; + my_iosys->ioroot = my_iosys->ioranks[0]; /* All the processes in this component, and the IO component, * are part of the union_comm. */ - if (cmp) + if (in_io || in_cmp) { - if (in_io || in_cmp) - { - LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp-1])); - /* Create a group for the union of the IO component - * and one of the computation components. */ - if ((ret = MPI_Comm_create(world, union_group[cmp - 1], - &my_iosys->union_comm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp])); + /* Create a group for the union of the IO component + * and one of the computation components. */ + if ((ret = MPI_Comm_create(world, union_group[cmp], &my_iosys->union_comm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank))) + return check_mpi(NULL, ret, __FILE__, __LINE__); - /* Set my_comm to union_comm for async. */ - my_iosys->my_comm = my_iosys->union_comm; - LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d", - cmp, my_iosys->union_rank, my_iosys->union_comm)); + /* Set my_comm to union_comm for async. */ + my_iosys->my_comm = my_iosys->union_comm; + LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d", + cmp, my_iosys->union_rank, my_iosys->union_comm)); - if (in_io) - { - LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm)); - /* Create the intercomm from IO to computation component. */ - LOG((3, "about to create intercomm for IO component to cmp = %d " - "my_iosys->io_comm = %d", cmp, my_iosys->io_comm)); - if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm, - my_proc_list[cmp][0], 0, &my_iosys->intercomm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - } - else - { - /* Create the intercomm from computation component to IO component. */ - LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp, - my_iosys->comp_comm)); - if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm, - my_proc_list[0][0], 0, &my_iosys->intercomm))) - return check_mpi(NULL, ret, __FILE__, __LINE__); - } - LOG((3, "intercomm created for cmp = %d", cmp)); + if (in_io) + { + LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm)); + /* Create the intercomm from IO to computation component. */ + LOG((3, "about to create intercomm for IO component to cmp = %d " + "my_iosys->io_comm = %d", cmp, my_iosys->io_comm)); + if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm, + my_proc_list[cmp][0], 0, &my_iosys->intercomm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); } - - /* Add this id to the list of PIO iosystem ids. */ - iosysidp[cmp - 1] = pio_add_to_iosystem_list(my_iosys); - LOG((2, "new iosys ID added to iosystem_list iosysid = %d\n", iosysidp[cmp - 1])); + else + { + /* Create the intercomm from computation component to IO component. */ + LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp, + my_iosys->comp_comm)); + if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm, + my_io_proc_list[0], 0, &my_iosys->intercomm))) + return check_mpi(NULL, ret, __FILE__, __LINE__); + } + LOG((3, "intercomm created for cmp = %d", cmp)); } - } + + /* Add this id to the list of PIO iosystem ids. */ + iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); + LOG((2, "new iosys ID added to iosystem_list iosysid = %d", iosysidp[cmp])); + } /* next computational component */ /* Now call the function from which the IO tasks will not return * until the PIO_MSG_EXIT message is sent. This will handle all @@ -1415,7 +1493,6 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, } /* Free resources if needed. */ - LOG((2, "PIOc_Init_Async starting to free resources")); if (!io_proc_list) free(my_io_proc_list); @@ -1425,7 +1502,7 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, if (!proc_list) { - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) free(my_proc_list[cmp]); free(my_proc_list); } @@ -1434,13 +1511,12 @@ int PIOc_Init_Async(MPI_Comm world, int num_io_procs, int *io_proc_list, if ((ret = MPI_Group_free(&io_group))) return check_mpi(NULL, ret, __FILE__, __LINE__); - for (int cmp = 0; cmp < component_count + 1; cmp++) + for (int cmp = 0; cmp < component_count; cmp++) { if ((ret = MPI_Group_free(&group[cmp]))) return check_mpi(NULL, ret, __FILE__, __LINE__); - if (cmp) - if ((ret = MPI_Group_free(&union_group[cmp - 1]))) - return check_mpi(NULL, ret, __FILE__, __LINE__); + if ((ret = MPI_Group_free(&union_group[cmp]))) + return check_mpi(NULL, ret, __FILE__, __LINE__); } if ((ret = MPI_Group_free(&world_group))) diff --git a/src/externals/pio2/src/clib/pioc_sc.c b/src/externals/pio2/src/clib/pioc_sc.c index 6b991b373dbd..98e3c6aa11a7 100644 --- a/src/externals/pio2/src/clib/pioc_sc.c +++ b/src/externals/pio2/src/clib/pioc_sc.c @@ -74,11 +74,11 @@ int gcd_array(int nain, int *ain) } /** - * Return the gcd of nain and any value in ain as int_64. + * Return the greatest common devisor of array ain as int_64. * - * @param main - * @param ain - * @returns + * @param nain number of elements in ain. + * @param ain array of length nain. + * @returns GCD of elements in ain. */ long long lgcd_array(int nain, long long *ain) { @@ -101,35 +101,50 @@ long long lgcd_array(int nain, long long *ain) } /** - * Compute start and count arrays. + * Compute one element (dimension) of start and count arrays. This + * function is used by CalcStartandCount(). * - * @param gdim - * @param ioprocs - * @param rank - * @param start - * @param kount + * @param gdim global size of one dimension. + * @param ioprocs number of io tasks. + * @param rank IO rank of this task. + * @param start pointer to PIO_Offset that will get the start value. + * @param count pointer to PIO_Offset that will get the count value. */ -void computestartandcount(int gdim, int ioprocs, int rank, PIO_Offset *start, - PIO_Offset *kount) +void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, + PIO_Offset *count) { - int irank; + int irank; /* The IO rank for this task. */ int remainder; int adds; - PIO_Offset lstart, lkount; + PIO_Offset lstart, lcount; + /* Check inputs. */ + pioassert(gdim >= 0 && ioprocs > 0 && rank >= 0 && start && count, + "invalid input", __FILE__, __LINE__); + + /* Determin which IO task to use. */ irank = rank % ioprocs; - lkount = (long int)(gdim / ioprocs); - lstart = (long int)(lkount * irank); - remainder = gdim - lkount * ioprocs; - if (remainder >= ioprocs-irank) + /* Each IO task will have its share of the global dim. */ + lcount = (long int)(gdim / ioprocs); + + /* Find the start for this task. */ + lstart = (long int)(lcount * irank); + + /* Is there anything left over? */ + remainder = gdim - lcount * ioprocs; + + /* Distribute left over data to some IO tasks. */ + if (remainder >= ioprocs - irank) { - lkount++; + lcount++; if ((adds = irank + remainder - ioprocs) > 0) lstart += adds; } + + /* Return results to caller. */ *start = lstart; - *kount = lkount; + *count = lcount; } /** @@ -138,20 +153,29 @@ void computestartandcount(int gdim, int ioprocs, int rank, PIO_Offset *start, * * @param arrlen * @param arr_in - * @returns + * @returns the size of the block */ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) { - int i, j, n, numblks, numtimes, ii, numgaps; - PIO_Offset bsize, bsizeg, blklensum; - PIO_Offset del_arr[arrlen - 1]; + int numblks = 0; /* Number of blocks. */ + int numtimes = 0; /* Number of times adjacent arr_in elements differ by != 1. */ + int numgaps = 0; /* Number of gaps. */ + int j; /* Loop counter. */ + int ii; /* Loop counter. */ + int n; + PIO_Offset bsize; /* Size of the block. */ + PIO_Offset bsizeg; /* Size of gap block. */ + PIO_Offset blklensum; /* Sum of all block lengths. */ + PIO_Offset del_arr[arrlen - 1]; /* Array of deltas between adjacent elements in arr_in. */ PIO_Offset loc_arr[arrlen - 1]; - numblks = 0; - numgaps = 0; - numtimes = 0; + /* Check inputs. */ + pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__); - for (i = 0; i < arrlen - 1; i++) + /* Count the number of contiguous blocks in arr_in. If any if + these blocks is of size 1, we are done and can return. + Otherwise numtimes is the number of blocks. */ + for (int i = 0; i < arrlen - 1; i++) { del_arr[i] = arr_in[i + 1] - arr_in[i]; if (del_arr[i] != 1) @@ -162,76 +186,95 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) } } + /* If numtimes is 0 the all of the data in arr_in is contiguous + * and numblks=1. Not sure why I have three different variables + * here, seems like n,numblks and numtimes could be combined. */ numblks = numtimes + 1; if (numtimes == 0) n = numblks; else n = numtimes; + /* If numblks==1 then the result is arrlen and you can return. */ bsize = (PIO_Offset)arrlen; if (numblks > 1) { PIO_Offset blk_len[numblks]; PIO_Offset gaps[numtimes]; - if(numtimes > 0) + + /* If numblks > 1 then numtimes must be > 0 and this if block + * isn't needed. */ + if (numtimes > 0) { ii = 0; - for (i = 0; i < arrlen - 1; i++) + for (int i = 0; i < arrlen - 1; i++) if (del_arr[i] > 1) gaps[ii++] = del_arr[i] - 1; numgaps = ii; } j = 0; - for (i = 0; i < n; i++) + for (int i = 0; i < n; i++) loc_arr[i] = 1; - for (i = 0; i < arrlen - 1; i++) + for (int i = 0; i < arrlen - 1; i++) if(del_arr[i] != 1) loc_arr[j++] = i; blk_len[0] = loc_arr[0]; blklensum = blk_len[0]; - for(i = 1; i < numblks - 1; i++) + for(int i = 1; i < numblks - 1; i++) { blk_len[i] = loc_arr[i] - loc_arr[i - 1]; blklensum += blk_len[i]; } blk_len[numblks - 1] = arrlen - blklensum; + /* Get the GCD in blk_len array. */ bsize = lgcd_array(numblks, blk_len); + + /* I don't recall why i needed these next two blocks, I + * remember struggling to get this right in all cases and I'm + * afraid that the end result is that bsize is almost always + * 1. */ if (numgaps > 0) { bsizeg = lgcd_array(numgaps, gaps); bsize = lgcd(bsize, bsizeg); } - if(arr_in[0] > 0) + + /* ??? */ + if (arr_in[0] > 0) bsize = lgcd(bsize, arr_in[0]); } + return bsize; } /** - * Compute start and count values for each io task. + * Compute start and count values for each io task. This is used in + * PIOc_InitDecomp() for the box rearranger only. * - * @param basetype - * @param ndims - * @param gdims - * @param num_io_procs - * @param myiorank - * @param start - * @param kount + * @param pio_type the PIO data type used in this decompotion. + * @param ndims the number of dimensions in the variable, not + * including the unlimited dimension. + * @param gdims an array of global size of each dimension. + * @param num_io_procs the number of IO tasks. + * @param myiorank rank of this task in IO communicator. + * @param start array of length ndims with data start values. + * @param count array of length ndims with data count values. + * @param num_aiotasks the number of IO tasks used(?) + * @returns 0 for success, error code otherwise. */ -int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_procs, - int myiorank, PIO_Offset *start, PIO_Offset *kount) +int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs, + int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks) { - int minbytes; + int minbytes; int maxbytes; - int minblocksize; - int basesize; + int minblocksize; /* Like minbytes, but in data elements. */ + int basesize; /* Size in bytes of base data type. */ int use_io_procs; int i; - long int p; long int pgdims; bool converged; int iorank; @@ -239,50 +282,61 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc int tiorank; int ioprocs; int tioprocs; - int mystart[ndims], mykount[ndims]; + int mystart[ndims], mycount[ndims]; long int pknt; - long int tpsize=0; - + long int tpsize = 0; + int ret; + + /* Check inputs. */ + pioassert(pio_type > 0 && ndims > 0 && gdims && num_io_procs > 0 && start && count, + "invalid input", __FILE__, __LINE__); + LOG((1, "CalcStartandCount pio_type = %d ndims = %d num_io_procs = %d myiorank = %d", + pio_type, ndims, num_io_procs, myiorank)); + + /* We are trying to find start and count indices for each iotask + * such that each task has approximately blocksize data to write + * (read). The number of iotasks participating in the operation is + * blocksize/global_size. */ minbytes = blocksize - 256; - maxbytes = blocksize + 256; + maxbytes = blocksize + 256; - switch (basetype) - { - case PIO_INT: - basesize = sizeof(int); - break; - case PIO_REAL: - basesize = sizeof(float); - break; - case PIO_DOUBLE: - basesize = sizeof(double); - break; - default: - piodie("Invalid basetype ",__FILE__,__LINE__); - break; - } + /* Determine the size of the data type. */ + if ((ret = find_mpi_type(pio_type, NULL, &basesize))) + return ret; + + /* Determine the minimum block size. */ minblocksize = minbytes / basesize; + /* Find the total size of the data. */ pgdims = 1; for (i = 0; i < ndims; i++) pgdims *= (long int)gdims[i]; - p = pgdims; - use_io_procs = max(1, min((int)((float)p / (float)minblocksize + 0.5), num_io_procs)); + + /* Find the number of ioprocs that are needed so that we have + * blocksize data on each iotask*/ + use_io_procs = max(1, min((int)((float)pgdims / (float)minblocksize + 0.5), num_io_procs)); + + /* Initialize to 0. */ converged = 0; for (i = 0; i < ndims; i++) { mystart[i] = 0; - mykount[i] = 0; + mycount[i] = 0; } + /* Use_io_procs is the number of ioprocs that are needed so that + * we have blocksize data on each iotask, now find start and count + * values needed on each of these tasks. */ while (!converged) { + long int p; + for (iorank = 0; iorank < use_io_procs; iorank++) { for (i = 0; i < ndims; i++) { start[i] = 0; - kount[i] = gdims[i]; + count[i] = gdims[i]; } ldims = ndims - 1; p = basesize; @@ -298,7 +352,7 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc if (gdims[ldims] < use_io_procs) { - if (ldims > 0 && gdims[ldims-1] > use_io_procs) + if (ldims > 0 && gdims[ldims - 1] > use_io_procs) ldims--; else use_io_procs -= (use_io_procs % gdims[ldims]); @@ -310,8 +364,8 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc { if (gdims[i] >= ioprocs) { - computestartandcount(gdims[i], ioprocs, tiorank, start + i, kount + i); - if (start[i] + kount[i] > gdims[i] + 1) + compute_one_dim(gdims[i], ioprocs, tiorank, &start[i], &count[i]); + if (start[i] + count[i] > gdims[i] + 1) { piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__); } @@ -320,7 +374,7 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc { tioprocs = gdims[i]; tiorank = (iorank * tioprocs) / ioprocs; - computestartandcount(gdims[i], tioprocs, tiorank, start + i, kount + i); + compute_one_dim(gdims[i], tioprocs, tiorank, &start[i], &count[i]); ioprocs = ioprocs / tioprocs; tiorank = iorank % ioprocs; } @@ -332,13 +386,13 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc for (i = 0; i < ndims; i++) { mystart[i] = start[i]; - mykount[i] = kount[i]; + mycount[i] = count[i]; } } pknt = 1; for(i = 0; i < ndims; i++) - pknt *= kount[i]; + pknt *= count[i]; tpsize += pknt; @@ -360,12 +414,14 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc } } + /* On IO tasks, set the start/count arrays to computed values. On + * non-io tasks set start/count to zero. */ if (myiorank < use_io_procs) { for (i = 0; i < ndims; i++) { start[i] = mystart[i]; - kount[i] = mykount[i]; + count[i] = mycount[i]; } } else @@ -373,9 +429,12 @@ int CalcStartandCount(int basetype, int ndims, const int *gdims, int num_io_proc for (i = 0; i < ndims; i++) { start[i] = 0; - kount[i] = 0; + count[i] = 0; } } - return use_io_procs; + /* Return the number of IO procs used to the caller. */ + *num_aiotasks = use_io_procs; + + return PIO_NOERR; } diff --git a/src/externals/pio2/src/clib/pioc_support.c b/src/externals/pio2/src/clib/pioc_support.c index 8f45277a32fd..fba1ad9070ac 100644 --- a/src/externals/pio2/src/clib/pioc_support.c +++ b/src/externals/pio2/src/clib/pioc_support.c @@ -101,13 +101,14 @@ int PIOc_strerror(int pioerr, char *errmsg) */ int PIOc_set_log_level(int level) { - int ret; #if PIO_ENABLE_LOGGING /* Set the log level. */ pio_log_level = level; #if NETCDF_C_LOGGING_ENABLED + int ret; + /* If netcdf logging is available turn it on starting at level = 4. */ if (level > NC_LEVEL_DIFF) if ((ret = nc_set_log_level(level - NC_LEVEL_DIFF))) @@ -148,7 +149,7 @@ void pio_init_logging(void) /** * Finalize logging - close log files, if open. */ -void pio_finalize_logging(void ) +void pio_finalize_logging(void) { #if PIO_ENABLE_LOGGING pio_log_ref_cnt -= 1; @@ -288,22 +289,6 @@ void print_trace(FILE *fp) free(strings); } -/** - * Exit due to lack of memory. - * - * @param ios the iosystem description struct - * @param req amount of memory that was being requested - * @param fname name of code file where error occured - * @param line the line of code where the error occurred. - */ -void piomemerror(iosystem_desc_t *ios, size_t req, char *fname, int line) -{ - char msg[80]; - sprintf(msg, "out of memory requesting: %ld", req); - cn_buffer_report(ios, false); - piodie(msg, fname, line); -} - /** * Abort program and call MPI_Abort(). * @@ -532,145 +517,173 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fn } /** - * Allocate an region. + * Allocate a region struct, and initialize it. * - * ndims the number of dimensions for the data in this region. - * @returns a pointer to the newly allocated io_region struct. + * @param ios pointer to the IO system info, used for error + * handling. Ignored if NULL. + * @param ndims the number of dimensions for the data in this region. + * @param a pointer that gets a pointer to the newly allocated + * io_region struct. + * @returns 0 for success, error code otherwise. */ -io_region *alloc_region(int ndims) +int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp) { io_region *region; + /* Check inputs. */ + pioassert(ndims >= 0 && regionp, "invalid input", __FILE__, __LINE__); + LOG((1, "alloc_region2 ndims = %d sizeof(io_region) = %d", ndims, + sizeof(io_region))); + /* Allocate memory for the io_region struct. */ - if (!(region = bget(sizeof(io_region)))) - return NULL; + if (!(region = calloc(1, sizeof(io_region)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Allocate memory for the array of start indicies. */ - if (!(region->start = bget(ndims * sizeof(PIO_Offset)))) - { - brel(region); - return NULL; - } + if (!(region->start = calloc(ndims, sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Allocate memory for the array of counts. */ - if (!(region->count = bget(ndims * sizeof(PIO_Offset)))) - { - brel(region); - brel(region->start); - return NULL; - } - - region->loffset = 0; - region->next = NULL; - - /* Initialize start and count arrays to zero. */ - for (int i = 0; i < ndims; i++) - { - region->start[i] = 0; - region->count[i] = 0; - } + if (!(region->count = calloc(ndims, sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - return region; + /* Return pointer to new region to caller. */ + *regionp = region; + + return PIO_NOERR; } /** - * Find the MPI type for a PIO type. + * Given a PIO type, find the MPI type and the type size. * * @param pio_type a PIO type, PIO_INT, PIO_FLOAT, etc. * @param mpi_type a pointer to MPI_Datatype that will get the MPI - * type that coresponds to the PIO type. + * type that coresponds to the PIO type. Ignored if NULL. + * @param type_size a pointer to int that will get the size of the + * type, in bytes. (For example, 4 for PIO_INT). Ignored if NULL. * @returns 0 for success, error code otherwise. */ -int find_mpi_type(int pio_type, MPI_Datatype *mpi_type) +int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size) { - /* Check input. */ - pioassert(mpi_type, "invalid input", __FILE__, __LINE__); + MPI_Datatype my_mpi_type; + int my_type_size; /* Decide on the base type. */ switch(pio_type) { case PIO_BYTE: - *mpi_type = MPI_BYTE; + my_mpi_type = MPI_BYTE; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_CHAR: - *mpi_type = MPI_CHAR; + my_mpi_type = MPI_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_SHORT: - *mpi_type = MPI_SHORT; + my_mpi_type = MPI_SHORT; + my_type_size = NETCDF_SHORT_SIZE; break; case PIO_INT: - *mpi_type = MPI_INT; + my_mpi_type = MPI_INT; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_FLOAT: - *mpi_type = MPI_FLOAT; + my_mpi_type = MPI_FLOAT; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_DOUBLE: - *mpi_type = MPI_DOUBLE; + my_mpi_type = MPI_DOUBLE; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; #ifdef _NETCDF4 case PIO_UBYTE: - *mpi_type = MPI_UNSIGNED_CHAR; + my_mpi_type = MPI_UNSIGNED_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; case PIO_USHORT: - *mpi_type = MPI_UNSIGNED_SHORT; + my_mpi_type = MPI_UNSIGNED_SHORT; + my_type_size = NETCDF_SHORT_SIZE; break; case PIO_UINT: - *mpi_type = MPI_UNSIGNED; + my_mpi_type = MPI_UNSIGNED; + my_type_size = NETCDF_INT_FLOAT_SIZE; break; case PIO_INT64: - *mpi_type = MPI_LONG_LONG; + my_mpi_type = MPI_LONG_LONG; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; case PIO_UINT64: - *mpi_type = MPI_UNSIGNED_LONG_LONG; + my_mpi_type = MPI_UNSIGNED_LONG_LONG; + my_type_size = NETCDF_DOUBLE_INT64_SIZE; break; case PIO_STRING: - *mpi_type = MPI_CHAR; + my_mpi_type = MPI_CHAR; + my_type_size = NETCDF_CHAR_SIZE; break; #endif /* _NETCDF4 */ default: return PIO_EBADTYPE; } + /* If caller wants MPI type, set it. */ + if (mpi_type) + *mpi_type = my_mpi_type; + + /* If caller wants type size, set it. */ + if (type_size) + *type_size = my_type_size; + return PIO_NOERR; } /** - * Allocate space for an IO description struct. + * Allocate space for an IO description struct, and initialize it. * - * @param ios pointer to the IO system info. + * @param ios pointer to the IO system info, used for error + * handling. * @param piotype the PIO data type (ex. PIO_FLOAT, PIO_INT, etc.). * @param ndims the number of dimensions. - * @iodesc pointer that gets a pointer to the newly allocated - * io_desc_t or NULL if allocation failed. + * @param iodesc pointer that gets the newly allocated io_desc_t. * @returns 0 for success, error code otherwise. */ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc) { MPI_Datatype mpi_type; + int mpierr; int ret; /* Check input. */ - pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + pioassert(ios && piotype > 0 && ndims >= 0 && iodesc, + "invalid input", __FILE__, __LINE__); - /* Allocate space for the io_desc_t struct. */ - if (!(*iodesc = calloc(1, sizeof(io_desc_t)))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + LOG((1, "malloc_iodesc piotype = %d ndims = %d", piotype, ndims)); /* Get the MPI type corresponding with the PIO type. */ - if ((ret = find_mpi_type(piotype, &mpi_type))) + if ((ret = find_mpi_type(piotype, &mpi_type, NULL))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* Decide on the base type. */ + /* Allocate space for the io_desc_t struct. */ + if (!(*iodesc = calloc(1, sizeof(io_desc_t)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + /* Remember the MPI type. */ (*iodesc)->basetype = mpi_type; + /* Get the size of the type. */ + if ((mpierr = MPI_Type_size((*iodesc)->basetype, &(*iodesc)->basetype_size))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + /* Initialize some values in the struct. */ (*iodesc)->maxregions = 1; (*iodesc)->ioid = -1; (*iodesc)->ndims = ndims; - (*iodesc)->firstregion = alloc_region(ndims); - /* Set the swap memory settings to defaults. */ + /* Allocate space for, and initialize, the first region. */ + if ((ret = alloc_region2(ios, ndims, &((*iodesc)->firstregion)))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + + /* Set the swap memory settings to defaults for this IO system. */ (*iodesc)->rearr_opts = ios->rearr_opts; return PIO_NOERR; @@ -689,12 +702,12 @@ void free_region_list(io_region *top) while (ptr) { if (ptr->start) - brel(ptr->start); + free(ptr->start); if (ptr->count) - brel(ptr->count); + free(ptr->count); tptr = ptr; ptr = ptr->next; - brel(tptr); + free(tptr); } } @@ -718,7 +731,7 @@ int PIOc_freedecomp(int iosysid, int ioid) return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -754,7 +767,7 @@ int PIOc_freedecomp(int iosysid, int ioid) { for (int i = 0; i < iodesc->nrecvs; i++) if (iodesc->rtype[i] != PIO_DATATYPE_NULL) - if ((mpierr = MPI_Type_free(iodesc->rtype + i))) + if ((mpierr = MPI_Type_free(&iodesc->rtype[i]))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); free(iodesc->rtype); @@ -938,7 +951,6 @@ int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset * @param filename the filename to be used. * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE. * @param ioid the ID of the IO description. - * @param comm an MPI communicator. * @param title optial title attribute for the file. Must be less than * NC_MAX_NAME + 1 if provided. Ignored if NULL. * @param history optial history attribute for the file. Must be less @@ -948,12 +960,11 @@ int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset * @returns 0 for success, error code otherwise. */ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, - MPI_Comm comm, char *title, char *history, int fortran_order) + char *title, char *history, int fortran_order) { - iosystem_desc_t *ios; - io_desc_t *iodesc; - int npes; /* Size of this communicator. */ - int myrank; /* Rank of this task. */ + iosystem_desc_t *ios; /* IO system info. */ + io_desc_t *iodesc; /* Decomposition info. */ + int max_maplen; /* The maximum maplen used for any task. */ int mpierr; int ret; @@ -963,42 +974,41 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, /* Check inputs. */ if (!filename) - return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + if (title) + if (strlen(title) > PIO_MAX_NAME) + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + if (history) + if (strlen(history) > PIO_MAX_NAME) + return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d", filename, - iosysid, ioid)); + LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d " + "ios->num_comptasks = %d", filename, iosysid, ioid, ios->num_comptasks)); /* Get the IO desc, which describes the decomposition. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Get the communicator size and task rank. */ - if ((mpierr = MPI_Comm_size(comm, &npes))) - return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Comm_rank(comm, &myrank))) - return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - LOG((2, "npes = %d myrank = %d", npes, myrank)); + /* Allocate memory for array which will contain the length of the + * map on each task, for all computation tasks. */ + int task_maplen[ios->num_comptasks]; + LOG((3, "ios->num_comptasks = %d", ios->num_comptasks)); - /* Allocate memory for the nmaplen. On task 0, this will contain - * the length of the map on each task, for all tasks. */ - int task_maplen[npes]; - - /* Gather maplens from all tasks and fill the task_maplen array on - * all tasks. */ - if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT, comm))) + /* Gather maplens from all computation tasks and fill the + * task_maplen array on all tasks. */ + if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT, + ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - /* We will need to know the maximum maplen used for any task. */ - int max_maplen; - /* Find the max maxplen. */ - if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX, comm))) + if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX, + ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((3, "max_maplen = %d", max_maplen)); - /* 2D array that, on task 0, will hold all the map information for - * all tasks. */ - int full_map[npes][max_maplen]; + /* 2D array that will hold all the map information for all + * tasks. */ + int full_map[ios->num_comptasks][max_maplen]; /* Fill local array with my map. Use the fill value for unused */ /* elements at the end if max_maplen is longer than maplen. Also @@ -1009,22 +1019,22 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid, my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT; LOG((3, "my_map[%d] = %d", e, my_map[e])); } - - /* Gather my_map from all tasks and fill the full_map array. */ + + /* Gather my_map from all computation tasks and fill the full_map array. */ if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen, - MPI_INT, comm))) + MPI_INT, ios->comp_comm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - - for (int p = 0; p < npes; p++) + + for (int p = 0; p < ios->num_comptasks; p++) for (int e = 0; e < max_maplen; e++) LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e])); /* Write the netCDF decomp file. */ - if ((ret = pioc_write_nc_decomp_int(iosysid, filename, cmode, iodesc->ndims, iodesc->dimlen, npes, - task_maplen, (int *)full_map, title, history, fortran_order))) + if ((ret = pioc_write_nc_decomp_int(ios, filename, cmode, iodesc->ndims, iodesc->dimlen, + ios->num_comptasks, task_maplen, (int *)full_map, title, + history, fortran_order))) return ret; - return PIO_NOERR; } @@ -1060,8 +1070,6 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm int num_tasks_decomp; /* The number of tasks for this decomp. */ int size; /* Size of comm. */ int my_rank; /* Task rank in comm. */ - MPI_Datatype mpi_type; /* Will be used as the basetype in iodesc. */ - int mpi_type_int; /* int version of mpi_type. */ char source_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */ char version_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */ int mpierr; @@ -1078,12 +1086,6 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm LOG((1, "PIOc_read_nc_decomp filename = %s iosysid = %d pio_type = %d", filename, iosysid, pio_type)); - /* Get the MPI type. We need it as an int. */ - if ((ret = find_mpi_type(pio_type, &mpi_type))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - mpi_type_int = mpi_type; - LOG((2, "mpi_type = %d mpi_type_int = %d", mpi_type, mpi_type_int)); - /* Get the communicator size and task rank. */ if ((mpierr = MPI_Comm_size(comm, &size))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1130,7 +1132,7 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm /* Write the decomp information in netCDF. This is an internal * function. * - * @param iosysid the IO system ID. + * @param ios pointer to io system info. * @param filename the name the decomp file will have. * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE. * @param ndims number of dims in the data being described. @@ -1151,31 +1153,22 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm * ordering, 0 for C array ordering. * @returns 0 for success, error code otherwise. */ -int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int ndims, +int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims, int *global_dimlen, int num_tasks, int *task_maplen, int *map, const char *title, const char *history, int fortran_order) { - iosystem_desc_t *ios; int max_maplen = 0; int ncid; int ret; - /* Get the IO system info. */ - if (!(ios = pio_get_iosystem_from_id(iosysid))) - return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Check inputs. */ - if (!filename || !global_dimlen || !task_maplen) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - if (title) - if (strlen(title) > PIO_MAX_NAME) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - if (history) - if (strlen(history) > PIO_MAX_NAME) - return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + pioassert(ios && filename && global_dimlen && task_maplen && + (!title || strlen(title) <= PIO_MAX_NAME) && + (!history || strlen(history) <= PIO_MAX_NAME), "invalid input", + __FILE__, __LINE__); - LOG((2, "pioc_write_nc_decomp_int iosysid = %d filename = %s ndims = %d num_tasks = %d", - iosysid, filename, ndims, num_tasks)); + LOG((2, "pioc_write_nc_decomp_int filename = %s ndims = %d num_tasks = %d", filename, + ndims, num_tasks)); /* Find the maximum maplen. */ for (int t = 0; t < num_tasks; t++) @@ -1184,7 +1177,7 @@ int pioc_write_nc_decomp_int(int iosysid, const char *filename, int cmode, int n LOG((3, "max_maplen = %d", max_maplen)); /* Create the netCDF decomp file. */ - if ((ret = PIOc_create(iosysid, filename, cmode | NC_WRITE, &ncid))) + if ((ret = PIOc_create(ios->iosysid, filename, cmode | NC_WRITE, &ncid))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); /* Write an attribute with the version of this file. */ @@ -1407,7 +1400,7 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int * int max_maplen_in; if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen_in))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - LOG((3, "max_maplen_in = %d", version_in)); + LOG((3, "max_maplen_in = %d", max_maplen_in)); if (max_maplen) *max_maplen = max_maplen_in; @@ -1737,11 +1730,11 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena ios->io_rank == 0) file->do_io = 1; - LOG((2, "file->do_io = %d ios->async_interface = %d", file->do_io, ios->async_interface)); + LOG((2, "file->do_io = %d ios->async = %d", file->do_io, ios->async)); /* If async is in use, and this is not an IO task, bcast the * parameters. */ - if (ios->async_interface) + if (ios->async) { int msg = PIO_MSG_CREATE_FILE; size_t len = strlen(filename); @@ -1914,7 +1907,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena file->do_io = 1; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { int msg = PIO_MSG_OPEN_FILE; size_t len = strlen(filename); @@ -2134,7 +2127,7 @@ int pioc_change_def(int ncid, int is_enddef) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) + if (ios->async) { if (!ios->ioproc) { @@ -2223,147 +2216,6 @@ int iotype_is_valid(int iotype) return ret; } -/** - * Internal function to compare rearranger flow control options. - * - * @param opt pointer to rearranger flow control options to compare. - * @param exp_opt pointer to rearranger flow control options with - * expected values. - * @return true if values in opt == values in exp_opt, false - * otherwise. - */ -bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt, - const rearr_comm_fc_opt_t *exp_opt) -{ - bool is_same = true; - - assert(opt && exp_opt); - - if (opt->enable_hs != exp_opt->enable_hs) - { - LOG((1, "Warning rearranger enable_hs = %s, expected = %s", - opt->enable_hs ? "TRUE" : "FALSE", exp_opt->enable_hs ? "TRUE" : "FALSE")); - is_same = false; - } - - if (opt->enable_isend != exp_opt->enable_isend) - { - LOG((1, "Warning rearranger enable_isend = %s, expected = %s", - opt->enable_isend ? "TRUE" : "FALSE", exp_opt->enable_isend ? "TRUE" : "FALSE")); - is_same = false; - } - - if (opt->max_pend_req != exp_opt->max_pend_req) - { - LOG((1, "Warning rearranger max_pend_req = %d, expected = %d", - opt->max_pend_req, exp_opt->max_pend_req)); - is_same = false; - } - - return is_same; -} - -/** - * Internal function to compare rearranger options. - * - * @param rearr_opts pointer to rearranger options to compare - * @param exp_rearr_opts pointer to rearranger options with the - * expected value - * @return true if values in rearr_opts == values in exp_rearr_opts - * false otherwise - */ -bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts) -{ - bool is_same = true; - - assert(rearr_opts && exp_rearr_opts); - - if (rearr_opts->comm_type != exp_rearr_opts->comm_type) - { - LOG((1, "Warning rearranger comm_type = %d, expected = %d. ", rearr_opts->comm_type, - exp_rearr_opts->comm_type)); - is_same = false; - } - - if (rearr_opts->fcd != exp_rearr_opts->fcd) - { - LOG((1, "Warning rearranger fcd = %d, expected = %d. ", rearr_opts->fcd, - exp_rearr_opts->fcd)); - is_same = false; - } - - is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_comp2io), - &(exp_rearr_opts->comm_fc_opts_comp2io)); - is_same = is_same && cmp_rearr_comm_fc_opts(&(rearr_opts->comm_fc_opts_io2comp), - &(exp_rearr_opts->comm_fc_opts_io2comp)); - - return is_same; -} - -/** - * Internal function to reset rearranger opts in iosystem to valid values. - * The old default for max pending requests was DEF_P2P_MAXREQ = 64. - * - * @param ios pointer to iosystem descriptor - */ -void check_and_reset_rearr_opts(iosystem_desc_t *ios) -{ - /* Disable handshake/isend and set max_pend_req to unlimited */ - const rearr_comm_fc_opt_t def_comm_nofc_opts = - { false, false, PIO_REARR_COMM_UNLIMITED_PEND_REQ }; - /* Disable handshake /isend and set max_pend_req = 0 to turn off throttling */ - const rearr_comm_fc_opt_t def_coll_comm_fc_opts = { false, false, 0 }; - const rearr_opt_t def_coll_rearr_opts = { - PIO_REARR_COMM_COLL, - PIO_REARR_COMM_FC_2D_DISABLE, - def_coll_comm_fc_opts, - def_coll_comm_fc_opts - }; - - assert(ios); - - /* Reset to defaults, if needed (user did not set it correctly) */ - if (ios->rearr_opts.comm_type == PIO_REARR_COMM_COLL) - { - /* Compare and log the user and default rearr opts for coll. */ - cmp_rearr_opts(&(ios->rearr_opts), &def_coll_rearr_opts); - /* Hard reset flow control options. */ - ios->rearr_opts = def_coll_rearr_opts; - } - else if (ios->rearr_opts.comm_type == PIO_REARR_COMM_P2P) - { - if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_2D_DISABLE) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io), - &def_comm_nofc_opts); - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp), - &def_comm_nofc_opts); - /* Hard reset flow control opts to defaults. */ - ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts; - ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts; - } - else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_COMP2IO) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_io2comp), - &def_comm_nofc_opts); - /* Hard reset io2comp dir to defaults. */ - ios->rearr_opts.comm_fc_opts_io2comp = def_comm_nofc_opts; - } - else if (ios->rearr_opts.fcd == PIO_REARR_COMM_FC_1D_IO2COMP) - { - /* Compare and log user and default opts. */ - cmp_rearr_comm_fc_opts(&(ios->rearr_opts.comm_fc_opts_comp2io), - &def_comm_nofc_opts); - /* Hard reset comp2io dir to defaults. */ - ios->rearr_opts.comm_fc_opts_comp2io = def_comm_nofc_opts; - } - /* Don't reset if flow control is enabled in both directions - * by user. */ - } -} - /** * Set the rearranger options associated with an iosystem * @@ -2404,21 +2256,25 @@ int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i, int max_pend_req_i2c) { iosystem_desc_t *ios; - int ret = PIO_NOERR; rearr_opt_t user_rearr_opts = { comm_type, fcd, {enable_hs_c2i,enable_isend_c2i, max_pend_req_c2i}, {enable_hs_i2c, enable_isend_i2c, max_pend_req_i2c} }; + /* Check inputs. */ + if ((comm_type != PIO_REARR_COMM_P2P && comm_type != PIO_REARR_COMM_FC_1D_COMP2IO) || + (fcd < 0 || fcd > PIO_REARR_COMM_FC_2D_DISABLE) || + (max_pend_req_c2i != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_c2i < 0) || + (max_pend_req_i2c != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_i2c < 0)) + return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); + /* Get the IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + /* Set the options. */ ios->rearr_opts = user_rearr_opts; - /* Perform sanity checks on the user supplied values */ - check_and_reset_rearr_opts(ios); - - return ret; + return PIO_NOERR; } diff --git a/src/externals/pio2/tests/cunit/CMakeLists.txt b/src/externals/pio2/tests/cunit/CMakeLists.txt index 1ef8be7afc3b..418340b79165 100644 --- a/src/externals/pio2/tests/cunit/CMakeLists.txt +++ b/src/externals/pio2/tests/cunit/CMakeLists.txt @@ -65,33 +65,45 @@ if (NOT PIO_USE_MPISERIAL) target_link_libraries (test_pioc_fill pioc) add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c) target_link_libraries (test_darray pioc) + add_executable (test_darray_multi EXCLUDE_FROM_ALL test_darray_multi.c test_common.c) + target_link_libraries (test_darray_multi pioc) add_executable (test_darray_multivar EXCLUDE_FROM_ALL test_darray_multivar.c test_common.c) target_link_libraries (test_darray_multivar pioc) + add_executable (test_darray_multivar2 EXCLUDE_FROM_ALL test_darray_multivar2.c test_common.c) + target_link_libraries (test_darray_multivar2 pioc) add_executable (test_darray_1d EXCLUDE_FROM_ALL test_darray_1d.c test_common.c) target_link_libraries (test_darray_1d pioc) add_executable (test_darray_3d EXCLUDE_FROM_ALL test_darray_3d.c test_common.c) target_link_libraries (test_darray_3d pioc) + add_executable (test_decomp_uneven EXCLUDE_FROM_ALL test_decomp_uneven.c test_common.c) + target_link_libraries (test_decomp_uneven pioc) add_executable (test_decomps EXCLUDE_FROM_ALL test_decomps.c test_common.c) target_link_libraries (test_decomps pioc) + add_executable (test_rearr EXCLUDE_FROM_ALL test_rearr.c test_common.c) + target_link_libraries (test_rearr pioc) endif () add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c) target_link_libraries (test_spmd pioc) add_dependencies (tests test_spmd) +add_dependencies (tests test_rearr) add_dependencies (tests test_pioc) add_dependencies (tests test_pioc_unlim) add_dependencies (tests test_pioc_putget) add_dependencies (tests test_pioc_fill) add_dependencies (tests test_darray) +add_dependencies (tests test_darray_multi) add_dependencies (tests test_darray_multivar) +add_dependencies (tests test_darray_multivar2) add_dependencies (tests test_darray_1d) add_dependencies (tests test_darray_3d) +add_dependencies (tests test_decomp_uneven) add_dependencies (tests test_decomps) # Test Timeout in seconds. if (PIO_VALGRIND_CHECK) - set (DEFAULT_TEST_TIMEOUT 120) + set (DEFAULT_TEST_TIMEOUT 240) else () - set (DEFAULT_TEST_TIMEOUT 60) + set (DEFAULT_TEST_TIMEOUT 120) endif () # All tests need a certain number of tasks, but they should be able to @@ -109,6 +121,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_spmd NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_rearr + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_rearr + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_intercomm2 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm2 NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -169,10 +185,18 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_darray_multi + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multi + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_darray_multivar EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_darray_multivar2 + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar2 + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_darray_1d EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_1d NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -181,6 +205,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_decomp_uneven + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomp_uneven + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_decomps EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomps NUMPROCS ${AT_LEAST_FOUR_TASKS} diff --git a/src/externals/pio2/tests/cunit/pio_tests.h b/src/externals/pio2/tests/cunit/pio_tests.h index 86ac35af5472..5a601ea6df17 100644 --- a/src/externals/pio2/tests/cunit/pio_tests.h +++ b/src/externals/pio2/tests/cunit/pio_tests.h @@ -29,6 +29,9 @@ /* Number of NetCDF-4 types. */ #define NUM_NETCDF4_TYPES 12 +/* Number of PIO rearrangers. */ +#define NUM_REARRANGERS 2 + /* Number of sample files constructed for these tests. */ #define NUM_SAMPLES 3 diff --git a/src/externals/pio2/tests/cunit/test_async_2comp.c b/src/externals/pio2/tests/cunit/test_async_2comp.c deleted file mode 100644 index 3c79cb6101e6..000000000000 --- a/src/externals/pio2/tests/cunit/test_async_2comp.c +++ /dev/null @@ -1,132 +0,0 @@ -/** - * @file Tests for PIOc_Intercomm. This tests basic asynch I/O capability. - * @author Ed Hartnett - * - * To run with valgrind, use this command: - * 
mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
- * --error-exitcode=99 --track-origins=yes ./test_intercomm3
- * - */ -#include -#include - -/* Number of processors that will do IO. */ -#define NUM_IO_PROCS 2 - -/* Number of computational components to create. */ -#define COMPONENT_COUNT 2 - -/* The number of tasks this test should run on. */ -#define TARGET_NTASKS 4 - -/* The name of this test. */ -#define TEST_NAME "test_intercomm3" - -/** Run Tests for Init_Intercomm - * - */ -int -main(int argc, char **argv) -{ - int verbose = 1; - - /* Zero-based rank of processor. */ - int my_rank; - - /* Number of processors involved in current execution. */ - int ntasks; - - /* Different output flavors. */ - int flavor[NUM_FLAVORS]; - - int num_flavors; - - /* The ID for the parallel I/O system. */ - int iosysid[COMPONENT_COUNT]; - - /* The ncid of the netCDF file. */ - int ncid; - - /* The ID of the netCDF varable. */ - int varid; - - /* Return code. */ - int ret; - - /* Index for loops. */ - int fmt, d, d1, i; - - /* Initialize test. */ - if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS))) - ERR(ERR_INIT); - - /* Figure out iotypes. */ - if ((ret = get_iotypes(&num_flavors, flavor))) - ERR(ret); - - /* How many processors will be used for our IO and 2 computation components. */ - int num_procs[COMPONENT_COUNT + 1] = {2, 1, 1}; - - /* Is the current process a computation task? */ - int comp_task = my_rank < 2 ? 0 : 1; - - /* Index of computation task in iosysid array. Varies by rank and - * does not apply to IO component processes. */ - int my_comp_idx = comp_task ? my_rank - 2 : -1; - - /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(MPI_COMM_WORLD, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, iosysid))) - ERR(ERR_AWFUL); - - /* All the netCDF calls are only executed on the computation - * tasks. The IO tasks have not returned from PIOc_Init_Intercomm, - * and when the do, they should go straight to finalize. */ - if (comp_task) - { - for (int flv = 0; flv < num_flavors; flv++) - { - char filename[NC_MAX_NAME + 1]; - - /* Create a filename. */ - int sample = 1; - sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, flavor_name(flv), sample, my_comp_idx); - - /* Create sample file 1. */ - printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename); - if ((ret = create_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL))) - ERR(ret); - - /* Check the file for correctness. */ - if ((ret = check_nc_sample_1(iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL))) - ERR(ret); - - } /* next netcdf format flavor */ - - /* If I don't sleep here for a second, there are problems. */ - sleep(2); - - /* Finalize the IO system. Only call this from the computation tasks. */ - if (verbose) - printf("%d test_intercomm3 Freeing PIO resources\n", my_rank); - for (int c = 0; c < COMPONENT_COUNT; c++) - { - if ((ret = PIOc_finalize(iosysid[c]))) - ERR(ret); - printf("%d test_intercomm3 PIOc_finalize completed for iosysid = %d\n", my_rank, iosysid[c]); - } - } /* endif comp_task */ - - /* Wait for everyone to catch up. */ - printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME); - MPI_Barrier(MPI_COMM_WORLD); - - /* Finalize the MPI library. */ - printf("%d %s Finalizing...\n", my_rank, TEST_NAME); - if ((ret = pio_test_finalize(&test_comm))) - return ERR_AWFUL; - - printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); - - return 0; -} diff --git a/src/externals/pio2/tests/cunit/test_async_3proc.c b/src/externals/pio2/tests/cunit/test_async_3proc.c index 020ed78172c0..4a2f8435fe2f 100644 --- a/src/externals/pio2/tests/cunit/test_async_3proc.c +++ b/src/externals/pio2/tests/cunit/test_async_3proc.c @@ -39,7 +39,7 @@ int main(int argc, char **argv) MPI_Comm test_comm; /* comm for test */ /* Num procs for IO and computation. */ - int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{2, 1}, {1, 2}}; + int num_procs[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}}; /* Number of processors that will do IO. */ int num_io_procs[NUM_COMBOS] = {2, 1}; @@ -63,8 +63,8 @@ int main(int argc, char **argv) int comp_task = my_rank < num_io_procs[combo] ? 0 : 1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, - num_procs[combo], NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, + num_procs[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < COMPONENT_COUNT; c++) diff --git a/src/externals/pio2/tests/cunit/test_async_4proc.c b/src/externals/pio2/tests/cunit/test_async_4proc.c index 4f91b0c0eecc..db0197ab9be9 100644 --- a/src/externals/pio2/tests/cunit/test_async_4proc.c +++ b/src/externals/pio2/tests/cunit/test_async_4proc.c @@ -32,15 +32,14 @@ int main(int argc, char **argv) int ret; /* Return code. */ MPI_Comm test_comm; - /* Num procs for IO and computation. */ - int num_procs[NUM_COMBOS][COMPONENT_COUNT + 1] = {{3, 1}, {2, 2}, {1, 3}}; + /* Num procs for computation. */ + int num_procs2[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}, {3}}; /* Number of processors that will do IO. */ int num_io_procs[NUM_COMBOS] = {3, 2, 1}; /* Initialize test. */ - if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, - &test_comm))) + if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm))) ERR(ERR_INIT); /* Test code runs on TARGET_NTASKS tasks. The left over tasks do @@ -57,8 +56,8 @@ int main(int argc, char **argv) int comp_task = my_rank < num_io_procs[combo] ? 0 : 1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, - num_procs[combo], NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT, + num_procs2[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < COMPONENT_COUNT; c++) diff --git a/src/externals/pio2/tests/cunit/test_async_simple.c b/src/externals/pio2/tests/cunit/test_async_simple.c index a0852b3e712d..4b4836bcb022 100644 --- a/src/externals/pio2/tests/cunit/test_async_simple.c +++ b/src/externals/pio2/tests/cunit/test_async_simple.c @@ -29,13 +29,18 @@ /* Run simple async test. */ int main(int argc, char **argv) { +#define NUM_IO_PROCS 1 +#define NUM_COMP_PROCS 1 int my_rank; /* Zero-based rank of processor. */ int ntasks; /* Number of processors involved in current execution. */ int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */ int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ int ret; /* Return code. */ - int num_procs[COMPONENT_COUNT + 1] = {1, 1}; /* Num procs for IO and computation. */ + int num_procs[COMPONENT_COUNT] = {1}; /* Num procs for IO and computation. */ + int io_proc_list[NUM_IO_PROCS] = {0}; + int comp_proc_list[NUM_COMP_PROCS] = {1}; + int *proc_list[COMPONENT_COUNT] = {comp_proc_list}; MPI_Comm test_comm; /* Initialize test. */ @@ -54,19 +59,22 @@ int main(int argc, char **argv) int comp_task = my_rank < NUM_IO_PROCS ? 0 : 1; /* Check for invalid values. */ - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, NULL) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, NULL) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, TEST_VAL_42, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, -1, - num_procs, NULL, NULL, NULL, iosysid) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, -1, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); - if (PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - NULL, NULL, NULL, NULL, iosysid) != PIO_EINVAL) + if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + NULL, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL) ERR(ERR_WRONG); /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT, + num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); /* All the netCDF calls are only executed on the computation diff --git a/src/externals/pio2/tests/cunit/test_darray_1d.c b/src/externals/pio2/tests/cunit/test_darray_1d.c index 01b894a87497..c5c6ac850125 100644 --- a/src/externals/pio2/tests/cunit/test_darray_1d.c +++ b/src/externals/pio2/tests/cunit/test_darray_1d.c @@ -107,18 +107,52 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * void *test_data_in; void *expected_in; PIO_Offset type_size; /* Size of the data type. */ - float my_float_rank = my_rank; /* my_rank in a float. */ - double my_double_rank = my_rank; /* my_rank in a double. */ + /* My rank as each type. */ + signed char my_byte_rank = my_rank; + char my_char_rank = my_rank; + short my_short_rank = my_rank; + float my_float_rank = my_rank; + double my_double_rank = my_rank; +#ifdef _NETCDF4 + unsigned char my_ubyte_rank = my_rank; + unsigned short my_ushort_rank = my_rank; + unsigned int my_uint_rank = my_rank; + long long my_int64_rank = my_rank; + unsigned long long my_uint64_rank = my_rank; +#endif /* _NETCDF4 */ + + /* Default fill value for each type. */ + signed char byte_fill = NC_FILL_BYTE; + char char_fill = NC_FILL_CHAR; + short short_fill = NC_FILL_SHORT; int int_fill = NC_FILL_INT; float float_fill = NC_FILL_FLOAT; double double_fill = NC_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = NC_FILL_UBYTE; + unsigned short ushort_fill = NC_FILL_USHORT; + unsigned int uint_fill = NC_FILL_UINT; + long long int64_fill = NC_FILL_INT64; + unsigned long long uint64_fill = NC_FILL_UINT64; +#endif /* _NETCDF4 */ + void *bufr; - int ret; /* Return code. */ + int ret; /* Return code. */ /* Use PIO to create the example file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + for (int with_fillvalue = 0; with_fillvalue < NUM_FILLVALUE_PRESENT_TESTS; with_fillvalue++) { /* Create the filename. */ @@ -152,11 +186,37 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * return ret; /* Initialize some data. */ + signed char byte_test_data[2] = {my_rank, my_rank}; + char char_test_data[2] = {my_rank, my_rank}; + short short_test_data[2] = {my_rank, my_rank}; int int_test_data[2] = {my_rank, my_rank}; float float_test_data[2] = {my_rank, my_rank}; double double_test_data[2] = {my_rank, my_rank}; +#ifdef _NETCDF4 + unsigned char ubyte_test_data[2] = {my_rank, my_rank}; + unsigned short ushort_test_data[2] = {my_rank, my_rank}; + unsigned int uint_test_data[2] = {my_rank, my_rank}; + long long int64_test_data[2] = {my_rank, my_rank}; + unsigned long long uint64_test_data[2] = {my_rank, my_rank}; +#endif /* _NETCDF4 */ + switch (pio_type) { + case PIO_BYTE: + test_data = byte_test_data; + fillvalue = with_fillvalue ? &byte_fill : NULL; + expected_in = &my_byte_rank; + break; + case PIO_CHAR: + test_data = char_test_data; + fillvalue = with_fillvalue ? &char_fill : NULL; + expected_in = &my_char_rank; + break; + case PIO_SHORT: + test_data = short_test_data; + fillvalue = with_fillvalue ? &short_fill : NULL; + expected_in = &my_short_rank; + break; case PIO_INT: test_data = int_test_data; fillvalue = with_fillvalue ? &int_fill : NULL; @@ -172,16 +232,42 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * fillvalue = with_fillvalue ? &double_fill : NULL; expected_in = &my_double_rank; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + test_data = ubyte_test_data; + fillvalue = with_fillvalue ? &ubyte_fill : NULL; + expected_in = &my_ubyte_rank; + break; + case PIO_USHORT: + test_data = ushort_test_data; + fillvalue = with_fillvalue ? &ushort_fill : NULL; + expected_in = &my_ushort_rank; + break; + case PIO_UINT: + test_data = uint_test_data; + fillvalue = with_fillvalue ? &uint_fill : NULL; + expected_in = &my_uint_rank; + break; + case PIO_INT64: + test_data = int64_test_data; + fillvalue = with_fillvalue ? &int64_fill : NULL; + expected_in = &my_int64_rank; + break; + case PIO_UINT64: + test_data = uint64_test_data; + fillvalue = with_fillvalue ? &uint64_fill : NULL; + expected_in = &my_uint64_rank; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } - /* Write the data. Our test_data contains only one real value - * (instead of 2, as indicated by arraylen), but due to the - * decomposition, only the first value is used in the - * output. */ - if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, - fillvalue))) + /* Write the data. Our test_data contains only one real + * value (instead of 2, as indicated by arraylen), but due + * to the decomposition, only the first value is used in + * the output. */ + if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); /* Close the netCDF file. */ @@ -221,6 +307,18 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * { switch (pio_type) { + case PIO_BYTE: + if (((signed char *)bufr)[e] != (e < 4 ? e : NC_FILL_BYTE)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (((char *)bufr)[e] != (e < 4 ? e : NC_FILL_CHAR)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (((short *)bufr)[e] != (e < 4 ? e : NC_FILL_SHORT)) + return ERR_WRONG; + break; case PIO_INT: if (((int *)bufr)[e] != (e < 4 ? e : NC_FILL_INT)) return ERR_WRONG; @@ -233,6 +331,28 @@ int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int * if (((double *)bufr)[e] != (e < 4 ? e : NC_FILL_DOUBLE)) return ERR_WRONG; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)bufr)[e] != (e < 4 ? e : NC_FILL_UBYTE)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (((unsigned short *)bufr)[e] != (e < 4 ? e : NC_FILL_USHORT)) + return ERR_WRONG; + break; + case PIO_UINT: + if (((unsigned int *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT)) + return ERR_WRONG; + break; + case PIO_INT64: + if (((long long *)bufr)[e] != (e < 4 ? e : NC_FILL_INT64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (((unsigned long long *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -277,11 +397,35 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, void *test_data_in; void *expected_in; PIO_Offset type_size; /* Size of the data type. */ - float my_float_rank = my_rank; /* my_rank in a float. */ - double my_double_rank = my_rank; /* my_rank in a double. */ + + /* My rank as each type. */ + signed char my_byte_rank = my_rank; + char my_char_rank = my_rank; + short my_short_rank = my_rank; + float my_float_rank = my_rank; + double my_double_rank = my_rank; +#ifdef _NETCDF4 + unsigned char my_ubyte_rank = my_rank; + unsigned short my_ushort_rank = my_rank; + unsigned int my_uint_rank = my_rank; + long long my_int64_rank = my_rank; + unsigned long long my_uint64_rank = my_rank; +#endif /* _NETCDF4 */ + + /* Default fill value for each type. */ + signed char byte_fill = NC_FILL_BYTE; + char char_fill = NC_FILL_CHAR; + short short_fill = NC_FILL_SHORT; int int_fill = NC_FILL_INT; float float_fill = NC_FILL_FLOAT; double double_fill = NC_FILL_DOUBLE; +#ifdef _NETCDF4 + unsigned char ubyte_fill = NC_FILL_UBYTE; + unsigned short ushort_fill = NC_FILL_USHORT; + unsigned int uint_fill = NC_FILL_UINT; + long long int64_fill = NC_FILL_INT64; + unsigned long long uint64_fill = NC_FILL_UINT64; +#endif /* _NETCDF4 */ void *bufr; int ret; /* Return code. */ @@ -289,6 +433,16 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + /* Create the filename. */ sprintf(filename, "data_%s_iotype_%d_pio_type_%d_unlim.nc", TEST_NAME, flavor[fmt], pio_type); @@ -322,11 +476,36 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, return ret; /* Initialize some data. */ + signed char byte_test_data[2] = {my_rank, my_rank}; + char char_test_data[2] = {my_rank, my_rank}; + short short_test_data[2] = {my_rank, my_rank}; int int_test_data[2] = {my_rank, my_rank}; float float_test_data[2] = {my_rank, my_rank}; double double_test_data[2] = {my_rank, my_rank}; +#ifdef _NETCDF4 + unsigned char ubyte_test_data[2] = {my_rank, my_rank}; + unsigned short ushort_test_data[2] = {my_rank, my_rank}; + unsigned int uint_test_data[2] = {my_rank, my_rank}; + long long int64_test_data[2] = {my_rank, my_rank}; + unsigned long long uint64_test_data[2] = {my_rank, my_rank}; +#endif /* _NETCDF4 */ switch (pio_type) { + case PIO_BYTE: + test_data = byte_test_data; + fillvalue = &byte_fill; + expected_in = &my_byte_rank; + break; + case PIO_CHAR: + test_data = char_test_data; + fillvalue = &char_fill; + expected_in = &my_char_rank; + break; + case PIO_SHORT: + test_data = short_test_data; + fillvalue = &short_fill; + expected_in = &my_short_rank; + break; case PIO_INT: test_data = int_test_data; fillvalue = &int_fill; @@ -342,6 +521,33 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, fillvalue = &double_fill; expected_in = &my_double_rank; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + test_data = ubyte_test_data; + fillvalue = &ubyte_fill; + expected_in = &my_ubyte_rank; + break; + case PIO_USHORT: + test_data = ushort_test_data; + fillvalue = &ushort_fill; + expected_in = &my_ushort_rank; + break; + case PIO_UINT: + test_data = uint_test_data; + fillvalue = &uint_fill; + expected_in = &my_uint_rank; + break; + case PIO_INT64: + test_data = int64_test_data; + fillvalue = &int64_fill; + expected_in = &my_int64_rank; + break; + case PIO_UINT64: + test_data = uint64_test_data; + fillvalue = &uint64_fill; + expected_in = &my_uint64_rank; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -406,6 +612,18 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, { switch (pio_type) { + case PIO_BYTE: + if (((signed char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_BYTE)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (((char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_CHAR)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (((short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_SHORT)) + return ERR_WRONG; + break; case PIO_INT: if (((int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT)) return ERR_WRONG; @@ -418,6 +636,28 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors, if (((double *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_DOUBLE)) return ERR_WRONG; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UBYTE)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (((unsigned short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_USHORT)) + return ERR_WRONG; + break; + case PIO_UINT: + if (((unsigned int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT)) + return ERR_WRONG; + break; + case PIO_INT64: + if (((long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (((unsigned long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -466,8 +706,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ @@ -480,10 +719,19 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, { iosystem_desc_t *ios; io_desc_t *iodesc; - int expected_basetype; + MPI_Datatype expected_basetype; switch (pio_type) { + case PIO_BYTE: + expected_basetype = MPI_BYTE; + break; + case PIO_CHAR: + expected_basetype = MPI_CHAR; + break; + case PIO_SHORT: + expected_basetype = MPI_SHORT; + break; case PIO_INT: expected_basetype = MPI_INT; break; @@ -493,6 +741,23 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, case PIO_DOUBLE: expected_basetype = MPI_DOUBLE; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + expected_basetype = MPI_UNSIGNED_CHAR; + break; + case PIO_USHORT: + expected_basetype = MPI_UNSIGNED_SHORT; + break; + case PIO_UINT: + expected_basetype = MPI_UNSIGNED; + break; + case PIO_INT64: + expected_basetype = MPI_LONG_LONG; + break; + case PIO_UINT64: + expected_basetype = MPI_UNSIGNED_LONG_LONG; + break; +#endif /* _NETCDF4 */ default: return ERR_WRONG; } @@ -536,8 +801,14 @@ int main(int argc, char **argv) { #define NUM_REARRANGERS_TO_TEST 2 int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; -#define NUM_TYPES_TO_TEST 3 - int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ int my_rank; int ntasks; int num_flavors; /* Number of PIO netCDF flavors in this build. */ diff --git a/src/externals/pio2/tests/cunit/test_darray_3d.c b/src/externals/pio2/tests/cunit/test_darray_3d.c index 07f2cd4e303c..e261c0cec433 100644 --- a/src/externals/pio2/tests/cunit/test_darray_3d.c +++ b/src/externals/pio2/tests/cunit/test_darray_3d.c @@ -92,7 +92,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + compdof, ioid, 0, NULL, NULL))) ERR(ret); printf("%d decomposition initialized.\n", my_rank); @@ -259,8 +259,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ diff --git a/src/externals/pio2/tests/cunit/test_darray_multi.c b/src/externals/pio2/tests/cunit/test_darray_multi.c new file mode 100644 index 000000000000..206b783e121b --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_darray_multi.c @@ -0,0 +1,473 @@ +/* + * Tests for PIO distributed arrays. This program tests the + * PIOc_write_darray_multi() function with more than one variable. + * + * Ed Hartnett, 3/7/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_multi" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* Number of variables. */ +#define NVAR 3 + +/* For attributes. */ +#define NOTE_NAME "note" +#define NOTE "This is a test file for the PIO library, and may be deleted." + +/* Who would have thought? */ +#define TOTAL_NUMBER_OF_STOOGES_NAME "Total_Number_of_Stooges" +#define TOTAL_NUMBER_OF_STOOGES 6 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"year", "Stooge_popularity", "face_smacks"}; + +/* The variable names. */ +char var_name[NVAR][PIO_MAX_NAME + 1] = {"Larry", "Curly", "Moe"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 3 variable, and use PIOc_write_darray_multi() to + * write one record of data to all three vars at once. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @returns 0 for success, error code otherwise. +*/ +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int pio_type) +{ +#define NUM_TEST_CASES_WRT_MULTI 2 +#define NUM_TEST_CASES_FILLVALUE 2 + + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int ncid2; /* The ncid of the re-opened netCDF file. */ + int varid[NVAR]; /* The IDs of the netCDF varables. */ + int ret; /* Return code. */ + PIO_Offset arraylen = 4; /* Amount of data from each task. */ + void *fillvalue; /* Pointer to fill value. */ + void *test_data; /* Pointer to test data we will write. */ + void *test_data_in; /* Pointer to buffer we will read into. */ + + /* Default fill value array for each type. */ + signed char byte_fill[NVAR] = {NC_FILL_BYTE, NC_FILL_BYTE, NC_FILL_BYTE}; + char char_fill[NVAR] = {NC_FILL_CHAR, NC_FILL_CHAR, NC_FILL_CHAR}; + short short_fill[NVAR] = {NC_FILL_SHORT, NC_FILL_SHORT, NC_FILL_SHORT}; + int int_fill[NVAR] = {NC_FILL_INT, NC_FILL_INT, NC_FILL_INT}; + float float_fill[NVAR] = {NC_FILL_FLOAT, NC_FILL_FLOAT, NC_FILL_FLOAT}; + double double_fill[NVAR] = {NC_FILL_DOUBLE, NC_FILL_DOUBLE, NC_FILL_DOUBLE}; +#ifdef _NETCDF4 + unsigned char ubyte_fill[NVAR] = {NC_FILL_UBYTE, NC_FILL_UBYTE, NC_FILL_UBYTE}; + unsigned short ushort_fill[NVAR] = {NC_FILL_USHORT, NC_FILL_USHORT, NC_FILL_USHORT}; + unsigned int uint_fill[NVAR] = {NC_FILL_UINT, NC_FILL_UINT, NC_FILL_UINT}; + long long int64_fill[NVAR] = {NC_FILL_INT64, NC_FILL_INT64, NC_FILL_INT64}; + unsigned long long uint64_fill[NVAR] = {NC_FILL_UINT64, NC_FILL_UINT64, NC_FILL_UINT64}; +#endif /* _NETCDF4 */ + + /* Test data we will write. */ + signed char test_data_byte[arraylen * NVAR]; + char test_data_char[arraylen * NVAR]; + short test_data_short[arraylen * NVAR]; + int test_data_int[arraylen * NVAR]; + float test_data_float[arraylen * NVAR]; + double test_data_double[arraylen * NVAR]; +#ifdef _NETCDF4 + unsigned char test_data_ubyte[arraylen * NVAR]; + unsigned short test_data_ushort[arraylen * NVAR]; + unsigned int test_data_uint[arraylen * NVAR]; + long long test_data_int64[arraylen * NVAR]; + unsigned long long test_data_uint64[arraylen * NVAR]; +#endif /* _NETCDF4 */ + + /* We will read test data into these buffers. */ + signed char test_data_byte_in[arraylen]; + char test_data_char_in[arraylen]; + short test_data_short_in[arraylen]; + int test_data_int_in[arraylen]; + float test_data_float_in[arraylen]; + double test_data_double_in[arraylen]; +#ifdef _NETCDF4 + unsigned char test_data_ubyte_in[arraylen]; + unsigned short test_data_ushort_in[arraylen]; + unsigned int test_data_uint_in[arraylen]; + long long test_data_int64_in[arraylen]; + unsigned long long test_data_uint64_in[arraylen]; +#endif /* _NETCDF4 */ + + /* Initialize a big blob of test data for NVAR vars. */ + for (int f = 0; f < arraylen * NVAR; f++) + { + test_data_byte[f] = my_rank * 1 + f; + test_data_char[f] = my_rank * 2 + f; + test_data_short[f] = my_rank * 5 + f; + test_data_int[f] = my_rank * 10 + f; + test_data_float[f] = my_rank * 10 + f + 0.5; + test_data_double[f] = my_rank * 100000 + f + 0.5; +#ifdef _NETCDF4 + test_data_ubyte[f] = my_rank * 3 + f; + test_data_ushort[f] = my_rank * 9 + f; + test_data_uint[f] = my_rank * 100 + f; + test_data_int64[f] = my_rank * 10000 + f; + test_data_uint64[f] = my_rank * 100000 + f; +#endif /* _NETCDF4 */ + } + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* 1-byte types not working with pnetcdf. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4. */ + if (pio_type > PIO_DOUBLE && (flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P)) + continue; + + /* Add a couple of extra tests for the + * PIOc_write_darray_multi() function. */ + for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++) + { + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME, + flavor[fmt], pio_type, test_multi, provide_fill); + + /* Select the fill value and data. */ + switch (pio_type) + { + case PIO_BYTE: + fillvalue = provide_fill ? byte_fill : NULL; + test_data = test_data_byte; + test_data_in = test_data_byte_in; + break; + case PIO_CHAR: + fillvalue = provide_fill ? char_fill : NULL; + test_data = test_data_char; + test_data_in = test_data_char_in; + break; + case PIO_SHORT: + fillvalue = provide_fill ? short_fill : NULL; + test_data = test_data_short; + test_data_in = test_data_short_in; + break; + case PIO_INT: + fillvalue = provide_fill ? int_fill : NULL; + test_data = test_data_int; + test_data_in = test_data_int_in; + break; + case PIO_FLOAT: + fillvalue = provide_fill ? float_fill : NULL; + test_data = test_data_float; + test_data_in = test_data_float_in; + break; + case PIO_DOUBLE: + fillvalue = provide_fill ? double_fill : NULL; + test_data = test_data_double; + test_data_in = test_data_double_in; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + fillvalue = provide_fill ? ubyte_fill : NULL; + test_data = test_data_ubyte; + test_data_in = test_data_ubyte_in; + break; + case PIO_USHORT: + fillvalue = provide_fill ? ushort_fill : NULL; + test_data = test_data_ushort; + test_data_in = test_data_ushort_in; + break; + case PIO_UINT: + fillvalue = provide_fill ? uint_fill : NULL; + test_data = test_data_uint; + test_data_in = test_data_uint_in; + break; + case PIO_INT64: + fillvalue = provide_fill ? int64_fill : NULL; + test_data = test_data_int64; + test_data_in = test_data_int64_in; + break; + case PIO_UINT64: + fillvalue = provide_fill ? uint64_fill : NULL; + test_data = test_data_uint64; + test_data_in = test_data_uint64_in; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + for (int v = 0; v < NVAR; v++) + if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + ERR(ret); + + /* Leave a note. */ + if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, NOTE_NAME, strlen(NOTE), NOTE))) + ERR(ret); + int num_stooges = TOTAL_NUMBER_OF_STOOGES; + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, TOTAL_NUMBER_OF_STOOGES_NAME, PIO_INT, 1, &num_stooges))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid[0], 0))) + ERR(ret); + + int frame[NVAR] = {0, 0, 0}; + int flushtodisk = test_multi; + + /* Write the data with the _multi function. */ + if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame, + fillvalue, flushtodisk))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + /* Now use read_darray on each var in turn and make + * sure we get correct data. */ + for (int v = 0; v < NVAR; v++) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != test_data_byte[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != test_data_char[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != test_data_short[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != test_data_float[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != test_data_double[f + arraylen * v]) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != test_data_ubyte[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != test_data_ushort[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != test_data_uint[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != test_data_int64[f + arraylen * v]) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != test_data_uint64[f + arraylen * v]) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + } + } + + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } /* next fillvalue test case */ + } /* next test multi */ + } /* next iotype */ + + return PIO_NOERR; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ + int ioid; + char filename[NC_MAX_NAME + 1]; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, + *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + printf("%d Running tests...\n", my_rank); + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar.c b/src/externals/pio2/tests/cunit/test_darray_multivar.c index c4b91c32ae24..773de9b5f600 100644 --- a/src/externals/pio2/tests/cunit/test_darray_multivar.c +++ b/src/externals/pio2/tests/cunit/test_darray_multivar.c @@ -71,10 +71,11 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; * (ignored if use_fill is 0). * @returns 0 for success, error code otherwise. */ -int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, - int pio_type, MPI_Comm test_comm, int rearranger, int use_fill, - int use_default) +int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int pio_type, MPI_Comm test_comm, + int rearranger, int use_fill, int use_default) { +#define NUM_MULTIVAR_TEST_CASES 2 char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ int ncid; /* The ncid of the netCDF file. */ @@ -84,6 +85,18 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran void *fillvalue; void *test_data; void *test_data_in; + signed char fillvalue_byte = NC_FILL_BYTE; + signed char custom_fillvalue_byte = -TEST_VAL_42; + signed char test_data_byte[arraylen]; + signed char test_data_byte_in[arraylen]; + char fillvalue_char = NC_FILL_CHAR; + char custom_fillvalue_char = -TEST_VAL_42; + char test_data_char[arraylen]; + char test_data_char_in[arraylen]; + short fillvalue_short = NC_FILL_SHORT; + short custom_fillvalue_short = -TEST_VAL_42; + short test_data_short[arraylen]; + short test_data_short_in[arraylen]; int fillvalue_int = NC_FILL_INT; int custom_fillvalue_int = -TEST_VAL_42; int test_data_int[arraylen]; @@ -96,19 +109,66 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran double custom_fillvalue_double = (-TEST_VAL_42 * 100); double test_data_double[arraylen]; double test_data_double_in[arraylen]; +#ifdef _NETCDF4 + unsigned char fillvalue_ubyte = NC_FILL_UBYTE; + unsigned char custom_fillvalue_ubyte = TEST_VAL_42; + unsigned char test_data_ubyte[arraylen]; + unsigned char test_data_ubyte_in[arraylen]; + unsigned short fillvalue_ushort = NC_FILL_USHORT; + unsigned short custom_fillvalue_ushort = (TEST_VAL_42 * 100); + unsigned short test_data_ushort[arraylen]; + unsigned short test_data_ushort_in[arraylen]; + unsigned int fillvalue_uint = NC_FILL_UINT; + unsigned int custom_fillvalue_uint = (TEST_VAL_42 * 100); + unsigned int test_data_uint[arraylen]; + unsigned int test_data_uint_in[arraylen]; + long long fillvalue_int64 = NC_FILL_INT64; + long long custom_fillvalue_int64 = (TEST_VAL_42 * 100); + long long test_data_int64[arraylen]; + long long test_data_int64_in[arraylen]; + unsigned long long fillvalue_uint64 = NC_FILL_UINT64; + unsigned long long custom_fillvalue_uint64 = (TEST_VAL_42 * 100); + unsigned long long test_data_uint64[arraylen]; + unsigned long long test_data_uint64_in[arraylen]; +#endif /* _NETCDF4 */ int ret; /* Return code. */ /* Initialize some data. */ for (int f = 0; f < arraylen; f++) { + test_data_byte[f] = my_rank * 10 + f; + test_data_char[f] = my_rank * 10 + f; + test_data_short[f] = my_rank * 10 + f; test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; test_data_double[f] = my_rank * 100000 + f + 0.5; +#ifdef _NETCDF4 + test_data_ubyte[f] = my_rank * 10 + f; + test_data_ushort[f] = my_rank * 10 + f; + test_data_uint[f] = my_rank * 10 + f; + test_data_int64[f] = my_rank * 10 + f; + test_data_uint64[f] = my_rank * 10 + f; +#endif /* _NETCDF4 */ } /* Select the fill value and data. */ switch (pio_type) { + case PIO_BYTE: + fillvalue = use_default ? &fillvalue_byte : &custom_fillvalue_byte; + test_data = test_data_byte; + test_data_in = test_data_byte_in; + break; + case PIO_CHAR: + fillvalue = use_default ? &fillvalue_char : &custom_fillvalue_char; + test_data = test_data_char; + test_data_in = test_data_char_in; + break; + case PIO_SHORT: + fillvalue = use_default ? &fillvalue_short : &custom_fillvalue_short; + test_data = test_data_short; + test_data_in = test_data_short_in; + break; case PIO_INT: fillvalue = use_default ? &fillvalue_int : &custom_fillvalue_int; test_data = test_data_int; @@ -124,6 +184,33 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran test_data = test_data_double; test_data_in = test_data_double_in; break; +#ifdef _NETCDF4 + case PIO_UBYTE: + fillvalue = use_default ? &fillvalue_ubyte : &custom_fillvalue_ubyte; + test_data = test_data_ubyte; + test_data_in = test_data_ubyte_in; + break; + case PIO_USHORT: + fillvalue = use_default ? &fillvalue_ushort : &custom_fillvalue_ushort; + test_data = test_data_ushort; + test_data_in = test_data_ushort_in; + break; + case PIO_UINT: + fillvalue = use_default ? &fillvalue_uint : &custom_fillvalue_uint; + test_data = test_data_uint; + test_data_in = test_data_uint_in; + break; + case PIO_INT64: + fillvalue = use_default ? &fillvalue_int64 : &custom_fillvalue_int64; + test_data = test_data_int64; + test_data_in = test_data_int64_in; + break; + case PIO_UINT64: + fillvalue = use_default ? &fillvalue_uint64 : &custom_fillvalue_uint64; + test_data = test_data_uint64; + test_data_in = test_data_uint64_in; + break; +#endif /* _NETCDF4 */ default: ERR(ERR_WRONG); } @@ -168,122 +255,226 @@ int test_3_empty(int iosysid, int ioid, int num_flavors, int *flavor, int my_ran #endif /* _PNETCDF */ } - /* Use PIO to create the example file in each of the four - * available ways. */ - for (int fmt = 0; fmt < num_flavors; fmt++) + for (int tc = 0; tc < NUM_MULTIVAR_TEST_CASES; tc++) { - /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc", - TEST_NAME, flavor[fmt], pio_type, use_fill, use_default); - - /* Create the netCDF output file. */ - printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, - flavor[fmt], pio_type); - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) - ERR(ret); - - /* Turn on fill mode if desired. */ - if (use_fill) - if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) - ERR(ret); - - /* Define netCDF dimensions and variable. */ - printf("%d Defining netCDF metadata...\n", my_rank); - for (int d = 0; d < NDIM; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) - ERR(ret); - - /* Define the variables. */ - for (int v = 0; v < NUM_VAR; v++) + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) { - if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + continue; + + /* NetCDF-4 types only work with netCDF-4 formats. */ + printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]); + if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && + flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_tc_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc", + TEST_NAME, flavor[fmt], tc, pio_type, use_fill, use_default); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); - if (use_fill && !use_default) - if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue))) - ERR(ret); - } - /* End define mode. */ - if ((ret = PIOc_enddef(ncid))) - ERR(ret); + /* Turn on fill mode if desired. */ + if (use_fill) + if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) + ERR(ret); - /* Set the value of the record dimension. */ - if ((ret = PIOc_setframe(ncid, varid[0], 0))) - ERR(ret); + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); - /* Write the data. */ - if ((ret = PIOc_write_darray(ncid, varid[0], ioid, arraylen, test_data, fillvalue))) - ERR(ret); + /* Define the variables. */ + for (int v = 0; v < NUM_VAR; v++) + { + if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v]))) + ERR(ret); + if (use_fill && !use_default) + if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue))) + ERR(ret); + } - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid))) - ERR(ret); + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); - /* Reopen the file. */ - if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) - ERR(ret); + /* Set the value of the record dimension. */ + for (int v = 0; v < NUM_VAR; v++) + { + if ((ret = PIOc_setframe(ncid, varid[v], 0))) + ERR(ret); - /* Read the data. */ - if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in))) - ERR(ret); + /* For the first test case we just write the first variable. */ + if (tc == 0) + break; + } - /* Check the results. */ - for (int f = 0; f < arraylen; f++) - { - switch (pio_type) + /* Write the data. */ + for (int v = 0; v < NUM_VAR; v++) { - case PIO_INT: - if (test_data_int_in[f] != test_data_int[f]) - return ERR_WRONG; - break; - case PIO_FLOAT: - if (test_data_float_in[f] != test_data_float[f]) - return ERR_WRONG; - break; - case PIO_DOUBLE: - if (test_data_double_in[f] != test_data_double[f]) - return ERR_WRONG; - break; - default: - ERR(ERR_WRONG); + if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data, fillvalue))) + ERR(ret); + + /* For the first test case we just write the first variable. */ + if (tc == 0) + break; } - } - /* If fill mode is in use the other vars should have fill values. */ - if (use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF) - { - /* Read the data. */ - if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in))) + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) ERR(ret); - - /* Check the results. */ - for (int f = 0; f < arraylen; f++) + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + for (int v = 0; v < NUM_VAR; v++) { - switch (pio_type) + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) { - case PIO_INT: - if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int)) - return ERR_WRONG; - break; - case PIO_FLOAT: - if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float)) - return ERR_WRONG; - break; - case PIO_DOUBLE: - if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double)) - return ERR_WRONG; + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != test_data_byte[f]) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != test_data_char[f]) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != test_data_short[f]) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != test_data_float[f]) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != test_data_double[f]) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != test_data_ubyte[f]) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != test_data_ushort[f]) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != test_data_uint[f]) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != test_data_int64[f]) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != test_data_uint64[f]) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } + } + + /* For the first test case we just write the first variable. */ + if (tc == 0) break; - default: - ERR(ERR_WRONG); + } /* next var */ + + /* For the first test case, if fill mode is in use the other vars + * should have fill values. */ + if (tc == 0 && use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_BYTE: + if (test_data_byte_in[f] != (use_default ? NC_FILL_BYTE : custom_fillvalue_byte)) + return ERR_WRONG; + break; + case PIO_CHAR: + if (test_data_char_in[f] != (use_default ? NC_FILL_CHAR : custom_fillvalue_char)) + return ERR_WRONG; + break; + case PIO_SHORT: + if (test_data_short_in[f] != (use_default ? NC_FILL_SHORT : custom_fillvalue_short)) + return ERR_WRONG; + break; + case PIO_INT: + if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int)) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float)) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double)) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (test_data_ubyte_in[f] != (use_default ? NC_FILL_UBYTE : custom_fillvalue_ubyte)) + return ERR_WRONG; + break; + case PIO_USHORT: + if (test_data_ushort_in[f] != (use_default ? NC_FILL_USHORT : custom_fillvalue_ushort)) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != (use_default ? NC_FILL_UINT : custom_fillvalue_uint)) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != (use_default ? NC_FILL_INT64 : custom_fillvalue_int64)) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != (use_default ? NC_FILL_UINT64 : custom_fillvalue_uint64)) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + ERR(ERR_WRONG); + } } } - } - /* Close the netCDF file. */ - printf("%d Closing the sample data file...\n", my_rank); - if ((ret = PIOc_closefile(ncid2))) - ERR(ret); - } + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } + } /* next test case */ + return PIO_NOERR; } @@ -302,8 +493,14 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm, int rearranger) { #define NUM_FILL_TESTS 3 -#define NUM_TYPES_TO_TEST 3 - int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ int ioid; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int ret; /* Return code. */ @@ -315,7 +512,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + &ioid, test_type[t]))) return ret; /* Run the different combinations of use_fill and use_default. */ @@ -327,9 +524,9 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, if (f == 2) use_default++; - /* Run a simple darray test. */ - if ((ret = test_3_empty(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t], - test_comm, rearranger, use_fill, use_default))) + /* Run the multivar darray tests. */ + if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, test_type[t], + test_comm, rearranger, use_fill, use_default))) return ret; } diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar2.c b/src/externals/pio2/tests/cunit/test_darray_multivar2.c new file mode 100644 index 000000000000..b25ba890ab7d --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_darray_multivar2.c @@ -0,0 +1,287 @@ +/* + * Tests for PIO distributed arrays. + * + * Ed Hartnett, Jim Edwards, 4/20/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_multivar2" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* Number of variables in the test file. */ +#define NUM_VAR 2 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/* The var names. */ +char var_name[NUM_VAR][PIO_MAX_NAME + 1] = {"Aubery", "Martin"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 2 variables. One of the vars uses the record + * dimension, the other does not. Then use darray to write to them. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @param test_comm the communicator that is running this test. + * @returns 0 for success, error code otherwise. +*/ +int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int pio_type, MPI_Comm test_comm) +{ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int varid[NUM_VAR]; /* The IDs of the netCDF varables. */ + PIO_Offset arraylen = 4; + int custom_fillvalue_int = -TEST_VAL_42; + int test_data_int[arraylen]; + int ret; /* Return code. */ + + /* Initialize some data. */ + for (int f = 0; f < arraylen; f++) + test_data_int[f] = my_rank * 10 + f; + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d.nc", TEST_NAME, flavor[fmt], pio_type); + + /* Create the netCDF output file. */ + printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename, + flavor[fmt], pio_type); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + printf("%d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Var 0 does not have a record dim, varid 1 is a record var. */ + if ((ret = PIOc_def_var(ncid, var_name[0], pio_type, NDIM - 1, &dimids[1], &varid[0]))) + ERR(ret); + if ((ret = PIOc_def_var(ncid, var_name[1], pio_type, NDIM, dimids, &varid[1]))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension for varid 1. */ + if ((ret = PIOc_setframe(ncid, varid[1], 0))) + ERR(ret); + + /* Write the data. */ + for (int v = 0; v < NUM_VAR; v++) + if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data_int, &custom_fillvalue_int))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Check the file contents. */ + { + int ncid2; /* The ncid of the re-opened netCDF file. */ + int test_data_int_in[arraylen]; + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + for (int v = 0; v < NUM_VAR; v++) + { + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_int_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + } /* next var */ + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } + } + + return PIO_NOERR; +} + +/* Create the decomposition to divide the 3-dimensional sample data + * between the 4 tasks. For the purposes of decomposition we are only + * concerned with 2 dimensions - we ignore the unlimited dimension. + * + * @param ntasks the number of available tasks + * @param my_rank rank of this task. + * @param iosysid the IO system ID. + * @param dim_len_2d an array of length 2 with the dim lengths. + * @param ioid a pointer that gets the ID of this decomposition. + * @param pio_type the data type to use for the decomposition. + * @returns 0 for success, error code otherwise. + **/ +int create_decomposition_2d_2(int ntasks, int my_rank, int iosysid, int *dim_len_2d, + int *ioid, int pio_type) +{ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + PIO_Offset *compdof; /* The decomposition mapping. */ + int ret; + + /* How many data elements per task? In this example we will end up + * with 4. */ + elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks; + + /* Allocate space for the decomposition array. */ + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + for (int i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i + 1; + + /* Create the PIO decomposition for this test. */ + printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); + if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe, + compdof, ioid, NULL, NULL, NULL))) + ERR(ret); + + printf("%d decomposition initialized.\n", my_rank); + + /* Free the mapping. */ + free(compdof); + + return 0; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ + int ioid; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d_2(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, PIO_INT))) + return ret; + + /* Run the multivar darray tests. */ + if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, PIO_INT, + test_comm))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS, + 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, PIO_REARR_BOX, &iosysid))) + return ret; + + /* Run tests. */ + printf("%d Running tests...\n", my_rank); + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_decomp_uneven.c b/src/externals/pio2/tests/cunit/test_decomp_uneven.c new file mode 100644 index 000000000000..b6d47004eff1 --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_decomp_uneven.c @@ -0,0 +1,380 @@ +/* + * Tests for PIO distributed arrays. This tests cases when arrays do + * not distribute evenly over the processors. + * + * Ed Hartnett, 3/6/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_uneven" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* This is for 3D data decompositions. */ +#define NDIM3 3 + +/* Create the decomposition to divide the 4-dimensional sample data + * between the 4 tasks. For the purposes of decomposition we are only + * concerned with 3 dimensions - we ignore the unlimited dimension. + * + * @param ntasks the number of available tasks + * @param my_rank rank of this task. + * @param iosysid the IO system ID. + * @param dim_len an array of length 3 with the dimension sizes. + * @param pio_type the type for this decomposition. + * @param ioid a pointer that gets the ID of this decomposition. + * lengths. + * @returns 0 for success, error code otherwise. + **/ +int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *dim_len, + int pio_type, int *ioid) +{ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + PIO_Offset remainder; /* Left over array elements. */ + PIO_Offset *compdof; /* The decomposition mapping. */ + PIO_Offset data_size = 1; + int ret; + + /* How many data elements per task? In this example we will end up + * with 4. */ + for (int d = 0; d < NDIM3; d++) + data_size *= dim_len[d]; + elements_per_pe = data_size / ntasks; + remainder = data_size % ntasks; + + /* Distribute the remaining elements. */ + if (my_rank < remainder) + elements_per_pe++; + printf("%d elements_per_pe = %lld remainder = %lld\n", my_rank, elements_per_pe, remainder); + + /* Allocate space for the decomposition array. */ + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + + /* Describe the decomposition. */ + for (int i = 0; i < elements_per_pe; i++) + { + int my_remainder = 0; + if (my_rank >= remainder) + my_remainder = remainder; + compdof[i] = my_rank * elements_per_pe + i + my_remainder; + printf("%d my_remainder = %d compdof[%d] = %lld\n", my_rank, i, my_remainder, compdof[i]); + } + + /* Create the PIO decomposition for this test. */ + printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); + if ((ret = PIOc_init_decomp(iosysid, pio_type, NDIM3, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL))) + ERR(ret); + + printf("%d decomposition initialized.\n", my_rank); + + /* Free the mapping. */ + free(compdof); + + return 0; +} + +/** + * Test the decomp read/write functionality. Given an ioid for a 3D + * decomposition, this function will write a decomp file, then read it + * in to ensure the correct values are read. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param rearranger the rearranger to use (PIO_REARR_BOX or + * PIO_REARR_SUBSET). + * @param test_comm the MPI communicator for this test. + * @param dim_len array of length 3 with dim lengths. + * @param expected_maplen pointer to array of length TARGET_NTASKS + * with the maplen we expect to get for each of the tasks running this + * test. + * @param pio_type the type we expect to be associated with + * this decomposition. + * @param full_maplen the length of the full map. + * @param pointer to expected map, an array of TARGET_NTASKS * + * max_maplen. + * @returns 0 for success, error code otherwise. +*/ +int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int rearranger, MPI_Comm test_comm, int *dim_len, int *expected_maplen, + int pio_type, int fill_maplen, int *expected_map) +{ +#define TEST_DECOMP_TITLE "Decomposition data for test_darray_uneven.c in PIO library." +#define TEST_DECOMP_HISTORY "This file may be deleted; it is for test purposes only: " + int ioid2; /* ID for decomp we read into. */ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + char title[] = TEST_DECOMP_TITLE; + char history[PIO_MAX_NAME + 1] = TEST_DECOMP_HISTORY; + char title_in[PIO_MAX_NAME + 1]; + char history_in[PIO_MAX_NAME + 1]; + int fortran_order_in; /* Indicates fortran vs. c order. */ + int ret; /* Return code. */ + + /* Use PIO to create the decomp file in one of the four + * available ways. */ + for (int fmt = 0; fmt < 1; fmt++) + { + /* Create the filename. */ + sprintf(filename, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc", TEST_NAME, pio_type, + dim_len[0], dim_len[1], dim_len[2]); + + /* Create history string. */ + strncat(history, filename, NC_MAX_NAME - strlen(TEST_DECOMP_HISTORY)); + + printf("writing decomp file %s\n", filename); + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, title, history, 0))) + return ret; + printf("about to check map with netCDF\n"); + + /* Open the decomposition file with netCDF. */ + int ncid_in; + int iotype = PIO_IOTYPE_NETCDF; + if ((ret = PIOc_openfile(iosysid, &ncid_in, &iotype, filename, NC_NOWRITE))) + return ret; + + /* Get the max maplen. */ + int max_maplen; + if ((ret = PIOc_get_att_int(ncid_in, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen))) + return ret; + printf("max_maplen = %d\n", max_maplen); + + /* Check dims. */ + PIO_Offset ndims_in; + if ((ret = PIOc_inq_dim(ncid_in, 0, NULL, &ndims_in))) + return ret; + if (ndims_in != NDIM3) + return ERR_WRONG; + PIO_Offset ntasks_in; + if ((ret = PIOc_inq_dim(ncid_in, 1, NULL, &ntasks_in))) + return ret; + if (ntasks_in != TARGET_NTASKS) + return ERR_WRONG; + + /* Check the maplen. */ + int maplen_varid; + int maplen_in[TARGET_NTASKS]; + if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAPLEN_VAR_NAME, &maplen_varid))) + return ret; + if ((ret = PIOc_get_var(ncid_in, maplen_varid, &maplen_in))) + return ret; + for (int t = 0; t < TARGET_NTASKS; t++) + { + printf("%d maplen_in[%d] = %d expected_maplen[%d] = %d\n", my_rank, t, maplen_in[t], t, expected_maplen[t]); + if (maplen_in[t] != expected_maplen[t]) + return ERR_WRONG; + } + + /* Check the map. */ + int map_varid; + int map_in[TARGET_NTASKS][max_maplen]; + if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAP_VAR_NAME, &map_varid))) + return ret; + if ((ret = PIOc_get_var(ncid_in, map_varid, (int *)&map_in))) + return ret; + printf("about to check map\n"); + for (int t = 0; t < TARGET_NTASKS; t++) + { + for (int e = 0; e < max_maplen; e++) + { + printf("%d t = %d e = %d map_in[t][e] = %d expected_map[t * max_maplen + e] = %d\n", + my_rank, t, e, map_in[t][e], expected_map[t * max_maplen + e]); + if (map_in[t][e] != expected_map[t * max_maplen + e]) + return ERR_WRONG; + } + } + + /* Close the decomposition file. */ + if ((ret = PIOc_closefile(ncid_in))) + return ret; + + /* Read the decomposition file into PIO. */ + printf("reading decomp file %s\n", filename); + if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, pio_type, + title_in, history_in, &fortran_order_in))) + return ret; + + /* Check the results. */ + { + iosystem_desc_t *ios; + io_desc_t *iodesc; + + /* Get the IO system info. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* Get the IO desc, which describes the decomposition. */ + if (!(iodesc = pio_get_iodesc_from_id(ioid2))) + return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* We need to find the MPI type we will expect to see in + * iodesc. */ + MPI_Datatype expected_mpi_type; + if ((ret = find_mpi_type(pio_type, &expected_mpi_type, NULL))) + return ret; + + /* Check values in iodesc. */ + printf("ioid2 = %d iodesc->ioid = %d iodesc->maplen = %d iodesc->ndims = %d " + "iodesc->ndof = %d iodesc->rearranger = %d iodesc->maxregions = %d " + "iodesc->needsfill = %d iodesc->basetype = %d expected_mpi_type = %d\n", + ioid2, iodesc->ioid, iodesc->maplen, iodesc->ndims, iodesc->ndof, + iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->basetype, + expected_mpi_type); + if (strcmp(title, title_in) || strcmp(history, history_in)) + return ERR_WRONG; + if (iodesc->ioid != ioid2 || iodesc->rearranger != rearranger || + iodesc->basetype != expected_mpi_type) + return ERR_WRONG; + if (iodesc->ndims != NDIM3) + return ERR_WRONG; + if (iodesc->maplen != expected_maplen[my_rank]) + return ERR_WRONG; + if (iodesc->ndims != NDIM3 || iodesc->ndof != expected_maplen[my_rank]) + return ERR_WRONG; + if (iodesc->needsfill) + return ERR_WRONG; + /* Don't forget to add 1! */ + for (int e = 0; e < iodesc->maplen; e++) + { + printf("%d e = %d max_maplen = %d iodesc->map[e] = %lld expected_map[my_rank * max_maplen + e] = %d\n", + my_rank, e, max_maplen, iodesc->map[e], expected_map[my_rank * max_maplen + e]); + if (iodesc->map[e] != expected_map[my_rank * max_maplen + e] + 1) + return ERR_WRONG; + } + for (int d = 0; d < NDIM3; d++) + if (iodesc->dimlen[d] != dim_len[d]) + return ERR_WRONG; + } + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid2))) + ERR(ret); + } + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +/* #define NUM_TYPES_TO_TEST 3 */ +/* int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; */ +#define NUM_TYPES_TO_TEST 1 + int test_type[NUM_TYPES_TO_TEST] = {PIO_INT}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int map_1x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + int map_2x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + int map_3x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}; + int map_1x3x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, 6, PIO_FILL_INT, 7, 8, PIO_FILL_INT}; + int map_1x2x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, PIO_FILL_INT}; +#define NUM_DIM_COMBOS_TO_TEST 5 + int dim_len[NUM_DIM_COMBOS_TO_TEST][NDIM3] = {{1, 4, 4}, + {2, 4, 4}, + {3, 4, 4}, + {1, 3, 3}, + {1, 2, 3}}; + int expected_maplen[NUM_DIM_COMBOS_TO_TEST][TARGET_NTASKS] = {{4, 4, 4, 4}, + {8, 8, 8, 8}, + {12, 12, 12, 12}, + {3, 2, 2, 2}, + {2, 2, 1, 1}}; + int *expected_map[NUM_DIM_COMBOS_TO_TEST] = {map_1x4x4, map_2x4x4, map_3x4x4, map_1x3x3, map_1x2x3}; + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + int ioid; /* Decomposition ID. */ + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests for each data type. */ + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++) + { + /* What is length of map for this combo? */ + int full_maplen = 1; + for (int d = 0; d < NDIM3; d++) + full_maplen *= dim_len[dc][d]; + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, dim_len[dc], + test_type[t], &ioid))) + return ret; + + /* Test decomposition read/write. */ + if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, + rearranger[r], test_comm, dim_len[dc], + expected_maplen[dc], test_type[t], full_maplen, + expected_map[dc]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + } + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_decomps.c b/src/externals/pio2/tests/cunit/test_decomps.c index 032d10de7b0f..96dc1a386e20 100644 --- a/src/externals/pio2/tests/cunit/test_decomps.c +++ b/src/externals/pio2/tests/cunit/test_decomps.c @@ -290,8 +290,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, decomp_file_type); printf("writing decomp file %s\n", filename); - if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, test_comm, NULL, - NULL, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ diff --git a/src/externals/pio2/tests/cunit/test_intercomm2.c b/src/externals/pio2/tests/cunit/test_intercomm2.c index 0a6f4a5b2ff0..e55c6e1ff2e8 100644 --- a/src/externals/pio2/tests/cunit/test_intercomm2.c +++ b/src/externals/pio2/tests/cunit/test_intercomm2.c @@ -253,7 +253,7 @@ int check_file(int iosysid, int format, char *filename, int my_rank) ERR(ERR_WRONG); if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, too_long_name, &atttype, &attlen) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_get_att(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &att_data) != PIO_EBADID) + if (PIOc_get_att(ncid, NC_GLOBAL, TEST_NAME, &att_data) != PIO_ENOTATT) ERR(ERR_WRONG); if (PIOc_get_att(ncid, NC_GLOBAL, NULL, &att_data) != PIO_EINVAL) ERR(ERR_WRONG); @@ -317,7 +317,7 @@ int main(int argc, char **argv) ERR(ret); /* How many processors will be used for our IO and 2 computation components. */ - int num_procs[COMPONENT_COUNT + 1] = {2, 2}; + int num_procs[COMPONENT_COUNT] = {2}; /* Is the current process a computation task? */ int comp_task = my_rank < 2 ? 0 : 1; @@ -327,8 +327,8 @@ int main(int argc, char **argv) int my_comp_idx = comp_task ? 0 : -1; /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - num_procs, NULL, NULL, NULL, iosysid))) + if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid))) ERR(ERR_AWFUL); printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 comp task returned.\n", @@ -517,7 +517,6 @@ int main(int argc, char **argv) for (int i = 0; i < DIM_LEN; i++) data[i] = i; printf("%d test_intercomm2 writing data\n", my_rank); - printf("%d test_intercomm2 writing data\n", my_rank); start[0] = 0; count[0] = DIM_LEN; if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data))) diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c index 4cf217e72e53..66ae617372b1 100644 --- a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c +++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c @@ -97,6 +97,8 @@ int main(int argc, char **argv) ERR(ret); if (active) ERR(ERR_WRONG); + if ((ret = PIOc_iosystem_is_active(iosysid, NULL))) + ERR(ret); int numiotasks; if (PIOc_get_numiotasks(iosysid + TEST_VAL_42, &numiotasks) != PIO_EBADID) diff --git a/src/externals/pio2/tests/cunit/test_pioc.c b/src/externals/pio2/tests/cunit/test_pioc.c index 3cffd6454632..466ed99f7065 100644 --- a/src/externals/pio2/tests/cunit/test_pioc.c +++ b/src/externals/pio2/tests/cunit/test_pioc.c @@ -100,22 +100,23 @@ int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, int if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; - /* Describe the decomposition. This is a 1-based array, so add 1! */ + /* Describe the decomposition. The new init_decomp uses a 0-based + * array, so don't add 1! */ for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i + 1; + compdof[i] = my_rank * elements_per_pe + i; /* These should fail. */ - if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL) != PIO_EBADID) + if (PIOc_init_decomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL) != PIO_EINVAL) + if (PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL) != PIO_EINVAL) ERR(ERR_WRONG); /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe); - if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe, + compdof, ioid, 0, NULL, NULL))) ERR(ret); printf("%d decomposition initialized.\n", my_rank); @@ -315,7 +316,7 @@ int check_var_name(int my_rank, int ncid, MPI_Comm test_comm) * @param flavor the iotype * @param test_comm the MPI communicator of the test. * @param async 1 if we are testing async, 0 otherwise. - * @returns 0 for success, error code otherwise. + * @returns 0 for success, error code otherwise. */ int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async) { @@ -391,7 +392,7 @@ int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async) ERR(ret); if (att_int_value2 != ATT_VAL) return ERR_WRONG; - + /* Check second att. */ if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME2, &att_type, &att_len))) return ret; @@ -494,7 +495,7 @@ int test_iotypes(int my_rank) /* This is never present. */ if (PIOc_iotype_available(1000)) return ERR_WRONG; - + /* NetCDF is always present. */ if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF)) return ERR_WRONG; @@ -740,26 +741,26 @@ int define_metadata(int ncid, int my_rank, int flavor) return ERR_WRONG; if (PIOc_def_var_fill(ncid, varid, NC_FILL, NULL) != PIO_EINVAL) return ERR_WRONG; - + /* Set the fill value. */ if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, &int_fill))) return ret; - + /* These should not work. */ if (PIOc_inq_var_fill(ncid + TEST_VAL_42, varid, &fill_mode, &int_fill_in) != PIO_EBADID) return ERR_WRONG; if (PIOc_inq_var_fill(ncid, varid + TEST_VAL_42, &fill_mode, &int_fill_in) != PIO_ENOTVAR) return ERR_WRONG; - + /* Check the fill value. */ if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, &int_fill_in))) return ret; if (fill_mode != NC_FILL || int_fill_in != int_fill) ERR(ERR_WRONG); - + /* These should also work. */ int_fill_in = 0; - + /* This does not work for pnetcdf, but probably should. */ if (flavor != PIO_IOTYPE_PNETCDF) { @@ -886,10 +887,14 @@ int test_names(int iosysid, int num_flavors, int *flavor, int my_rank, return ERR_WRONG; if (PIOc_setframe(ncid, -1, 0) != PIO_EINVAL) return ERR_WRONG; + if (PIOc_setframe(ncid, NC_MAX_VARS + 1, 0) != PIO_EINVAL) + return ERR_WRONG; if (PIOc_advanceframe(ncid + TEST_VAL_42, 0) != PIO_EBADID) return ERR_WRONG; if (PIOc_advanceframe(ncid, -1) != PIO_EINVAL) return ERR_WRONG; + if (PIOc_advanceframe(ncid, NC_MAX_VARS + 1) != PIO_EINVAL) + return ERR_WRONG; /* Check the dimension names. */ if ((ret = check_dim_names(my_rank, ncid, test_comm))) @@ -1398,6 +1403,171 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank) return PIO_NOERR; } +/* This function is part of test_scalar(). It tests the contents of + * the scalar var. */ +int check_scalar_var(int ncid, int varid, int flavor) +{ + char var_name_in[PIO_MAX_NAME + 1]; + int var_type_in; + int ndims_in; + int natts_in; + int val_in; + int ret; + + /* Learn the var metadata. */ + if ((ret = PIOc_inq_var(ncid, varid, var_name_in, &var_type_in, &ndims_in, NULL, + &natts_in))) + return ret; + + /* Is the metadata correct? */ + if (strcmp(var_name_in, VAR_NAME) || var_type_in != PIO_INT || ndims_in != 0 || natts_in != 0) + return ERR_WRONG; + + /* Get the value. */ + if ((ret = PIOc_get_var_int(ncid, varid, &val_in))) + return ret; + printf("val_in = %d\n", val_in); + + /* Is the value correct? */ + if (val_in != TEST_VAL_42) + return ERR_WRONG; + + return 0; +} + +/* Test scalar vars. */ +int test_scalar(int iosysid, int num_flavors, int *flavor, int my_rank, int async, + MPI_Comm test_comm) +{ + int ncid; /* The ncid of the netCDF file. */ + int varid; /* The ID of the netCDF varable. */ + int ret; /* Return code. */ + + /* Use netCDF classic to create a file with a scalar var, then set + * and read the value. */ + if (my_rank == 0) + { + char test_file[] = "netcdf_test.nc"; + int test_val = TEST_VAL_42; + int test_val_in; + + if ((ret = nc_create(test_file, NC_CLOBBER, &ncid))) + return ret; + if ((ret = nc_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid))) + return ret; + if ((ret = nc_enddef(ncid))) + return ret; + if ((ret = nc_put_var(ncid, varid, &test_val))) + return ret; + if ((ret = nc_close(ncid))) + return ret; + if ((ret = nc_open(test_file, NC_NOWRITE, &ncid))) + return ret; + /* if ((ret = nc_get_var(ncid, varid, &test_val_in))) */ + /* return ret; */ + /* if (test_val_in != test_val) */ + /* return ERR_WRONG; */ + if ((ret = nc_get_vars(ncid, varid, NULL, NULL, NULL, &test_val_in))) + return ret; + if (test_val_in != test_val) + return ERR_WRONG; + if ((ret = nc_close(ncid))) + return ret; + } + + /* Use pnetCDF to create a file with a scalar var, then set and + * read the value. */ +#ifdef _PNETCDF + { + char test_file[] = "pnetcdf_test.nc"; + int test_val = TEST_VAL_42; + int test_val_in; + + if ((ret = ncmpi_create(test_comm, test_file, NC_CLOBBER, MPI_INFO_NULL, &ncid))) + return ret; + if ((ret = ncmpi_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid))) + return ret; + if ((ret = ncmpi_enddef(ncid))) + return ret; + if ((ret = ncmpi_put_var_int_all(ncid, varid, &test_val))) + return ret; + if ((ret = ncmpi_close(ncid))) + return ret; + if ((ret = ncmpi_open(test_comm, test_file, NC_NOWRITE, MPI_INFO_NULL, &ncid))) + return ret; + /* Turn on independent access for pnetcdf file. */ + if ((ret = ncmpi_begin_indep_data(ncid))) + return ret; + /* if ((ret = ncmpi_get_var_int(ncid, varid, &test_val_in))) */ + /* return ret; */ + if ((ret = ncmpi_get_vars_int(ncid, varid, NULL, NULL, NULL, &test_val_in))) + return ret; + if ((ret = ncmpi_end_indep_data(ncid))) + return ret; + if (test_val_in != test_val) + return ERR_WRONG; + printf("ret = %d test_val_in = %d\n", ret, test_val_in); + if (test_val_in != test_val) + return ERR_WRONG; + if ((ret = ncmpi_close(ncid))) + return ret; + } +#endif /* _PNETCDF */ + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + char filename[PIO_MAX_NAME + 1]; /* Test filename. */ + char iotype_name[PIO_MAX_NAME + 1]; + + /* Create a filename. */ + if ((ret = get_iotype_name(flavor[fmt], iotype_name))) + return ret; + sprintf(filename, "%s_%s_scalar_async_%d.nc", TEST_NAME, iotype_name, async); + + /* Create the netCDF output file. */ + printf("%d Creating test file %s.\n", my_rank, filename); + if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER))) + ERR(ret); + + /* Define a scalar variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, 0, NULL, &varid))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Write a scalar value. */ + int test_val = TEST_VAL_42; + if ((ret = PIOc_put_var_int(ncid, varid, &test_val))) + ERR(ret); + + /* Check the scalar var. */ + if ((ret = check_scalar_var(ncid, varid, flavor[fmt]))) + ERR(ret); + + /* Close the netCDF file. */ + printf("%d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE))) + ERR(ret); + + /* Check the scalar var again. */ + if ((ret = check_scalar_var(ncid, varid, flavor[fmt]))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + } + return PIO_NOERR; +} + /** Test the malloc_iodesc() function. * * @param my_rank rank of this task. @@ -1411,11 +1581,13 @@ int test_malloc_iodesc2(int iosysid, int my_rank) #else int num_types = NUM_CLASSIC_TYPES; #endif /* _NETCDF4 */ - int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, - PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; - int mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_FLOAT, MPI_DOUBLE, - MPI_UNSIGNED_CHAR, MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG, - MPI_UNSIGNED_LONG_LONG, MPI_CHAR}; + int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, + PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; + MPI_Datatype mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, + MPI_FLOAT, MPI_DOUBLE, MPI_UNSIGNED_CHAR, + MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG, + MPI_UNSIGNED_LONG_LONG, MPI_CHAR}; int ioid; iosystem_desc_t *ios; io_desc_t *iodesc; @@ -1423,10 +1595,11 @@ int test_malloc_iodesc2(int iosysid, int my_rank) if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - + printf("test_malloc_iodesc2 num_types %d\n",num_types); /* Test with each type. */ for (int t = 0; t < num_types; t++) { + if ((ret = malloc_iodesc(ios, test_type[t], 1, &iodesc))) return ret; if (iodesc->basetype != mpi_type[t]) @@ -1439,7 +1612,6 @@ int test_malloc_iodesc2(int iosysid, int my_rank) if ((ret = pio_delete_iodesc_from_list(ioid))) return ret; } - return 0; } @@ -1450,7 +1622,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len int ioid; char filename[NC_MAX_NAME + 1]; /* Test decomp filename. */ char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ - char too_long_name[PIO_MAX_NAME * 5 + 1]; + iosystem_desc_t *ios; /* IO system info. */ int ret; /* This will be our file name for writing out decompositions. */ @@ -1472,37 +1644,12 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len int task_maplen[TARGET_NTASKS] = {1, 1, 1, 1}; int map[TARGET_NTASKS][1] = {{0},{1},{2},{3}}; - /* These should not work. */ - memset(too_long_name, 74, PIO_MAX_NAME * 5); - too_long_name[PIO_MAX_NAME * 5] = 0; - if (pioc_write_nc_decomp_int(iosysid + TEST_VAL_42, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EBADID) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, NULL, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, NULL, - TARGET_NTASKS, task_maplen, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, NULL, (int *)map, title, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, too_long_name, - history, 0) != PIO_EINVAL) - return ERR_WRONG; - if (pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, - TARGET_NTASKS, task_maplen, (int *)map, title, - too_long_name, 0) != PIO_EINVAL) - return ERR_WRONG; - + /* Get the IO system info. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Write the decomposition file. */ - if ((ret = pioc_write_nc_decomp_int(iosysid, nc_filename, 0, NDIM1, global_dimlen, + if ((ret = pioc_write_nc_decomp_int(ios, nc_filename, 0, NDIM1, global_dimlen, TARGET_NTASKS, task_maplen, (int *)map, title, history, 0))) return ret; @@ -1515,7 +1662,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len char source_in[PIO_MAX_NAME + 1]; char version_in[PIO_MAX_NAME + 1]; char expected_source[] = "Decomposition file produced by PIO library."; - int *global_dimlen_in; + int *global_dimlen_in; int *task_maplen_in; int *map_in; int fortran_order_in; @@ -1540,6 +1687,7 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len history_in, source_in, version_in, &fortran_order_in))) return ret; + /* Did we get the correct answers? */ printf("source_in = %s\n", source_in); if (strcmp(title, title_in) || strcmp(history, history_in) || @@ -1650,7 +1798,6 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len free(task_maplen_in); free(map_in); - /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); @@ -1665,7 +1812,7 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, int ioid; char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ int ret; - + /* This will be our file name for writing out decompositions. */ sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); @@ -1679,15 +1826,29 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, char *history = "Added to PIO automatic testing by Ed in February 2017."; /* These should not work. */ - if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid, test_comm, title, history, 0) != PIO_EBADID) + char too_long_name[PIO_MAX_NAME * 5 + 1]; + memset(too_long_name, 74, PIO_MAX_NAME * 5); + too_long_name[PIO_MAX_NAME * 5] = 0; + + if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid, + title, history, 0) != PIO_EBADID) return ERR_WRONG; - if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, test_comm, title, history, 0) != PIO_EINVAL) + if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, title, history, 0) != PIO_EINVAL) return ERR_WRONG; - if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42, test_comm, title, history, 0) != PIO_EBADID) + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42, + title, history, 0) != PIO_EBADID) + return ERR_WRONG; + + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, + too_long_name, history, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, + title, too_long_name, 0) != PIO_EINVAL) return ERR_WRONG; /* Write a netCDF decomp file for this iosystem. */ - if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0))) + if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title, + history, 0))) return ret; int ioid_in; @@ -1696,8 +1857,8 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, int fortran_order_in; /* These should not work. */ - if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm, PIO_INT, - title_in, history_in, &fortran_order_in) != PIO_EBADID) + if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm, + PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EBADID) return ret; if (PIOc_read_nc_decomp(iosysid, NULL, &ioid_in, test_comm, PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EINVAL) @@ -1705,7 +1866,7 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, if (PIOc_read_nc_decomp(iosysid, nc_filename, NULL, test_comm, PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EINVAL) return ret; - + /* Read it using the public read function. */ if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_INT, title_in, history_in, &fortran_order_in))) @@ -1780,9 +1941,88 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len, free(map_in); /* /\* These should also work. *\/ */ - /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, test_comm, title, history, 0))) */ + /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title, history, 0))) */ /* return ret; */ + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions. */ +int test_decomp_public_2(int my_test_size, int my_rank, int iosysid, int dim_len, + MPI_Comm test_comm, int async) +{ + int ioid; + char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ + int ret; + + /* This will be our file name for writing out decompositions. */ + sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid))) + return ret; + + /* Write a netCDF decomp file for this iosystem. */ + if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, NULL, NULL, 0))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions. */ +int test_decomp_2(int my_test_size, int my_rank, int iosysid, int dim_len, + MPI_Comm test_comm, int async) +{ + int ioid; + char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */ + int ret; + + /* This will be our file name for writing out decompositions. */ + sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return 0; +} + +/* Test some decomp public API functions with async. */ +int test_decomp_public_async(int my_test_size, int my_rank, int iosysid, MPI_Comm test_comm, + int async) +{ +#define ELEM1 1 +#define LEN3 3 + int ioid; + int dim_len = LEN3; + PIO_Offset elements_per_pe = ELEM1; + PIO_Offset compdof[ELEM1] = {my_rank + 1}; + char filename[PIO_MAX_NAME + 1]; + int ret; + + sprintf(filename, "async_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async); + /* Create the PIO decomposition for this test. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, &dim_len, elements_per_pe, + compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + ERR(ret); + + /* Write the decomp file (on appropriate tasks). */ + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) + return ret; + /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); @@ -1797,6 +2037,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te int ioid; int my_test_size; char filename[NC_MAX_NAME + 1]; + char nc_filename[NC_MAX_NAME + 1]; int ret; /* Return code. */ if ((ret = MPI_Comm_size(test_comm, &my_test_size))) @@ -1804,7 +2045,14 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te /* This will be our file name for writing out decompositions. */ sprintf(filename, "decomp_%d.txt", my_rank); + sprintf(nc_filename, "decomp_%d.nc", my_rank); + /* This is a simple test that just creates the decomp with + * async. */ + if (async) + if ((ret = test_decomp_public_async(my_test_size, my_rank, iosysid, test_comm, async))) + return ret; + /* Check iotypes. */ printf("%d Testing iotypes. async = %d\n", my_rank, async); if ((ret = test_iotypes(my_rank))) @@ -1828,25 +2076,25 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te if (!async) if ((ret = test_decomp_internal(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; - /* Test decomposition public API functions. */ if (!async) if ((ret = test_decomp_public(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; + /* This is a simple test that just creates a decomp. */ + /* if ((ret = test_decomp_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) */ + /* return ret; */ + + /* This is a simple test that just writes the decomp. */ if (!async) - { - printf("%d Testing darray. async = %d\n", my_rank, async); - - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid))) + if ((ret = test_decomp_public_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) return ret; - /* Write out an ASCII version of the decomp file. */ - printf("%d Calling write_decomp. async = %d\n", my_rank, async); - if ((ret = PIOc_write_decomp(filename, iosysid, ioid, test_comm))) + /* Decompose the data over the tasks. */ + if (!async) + { + if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid))) return ret; - printf("%d Called write_decomp. async = %d\n", my_rank, async); /* Run the darray tests. */ for (int fv = 0; fv < 2; fv++) @@ -1872,6 +2120,11 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te printf("%d Testing nc4 functions. async = %d\n", my_rank, async); if ((ret = test_nc4(iosysid, num_flavors, flavor, my_rank))) return ret; + + /* Test scalar var. */ + printf("%d Testing scalar var. async = %d\n", my_rank, async); + if ((ret = test_scalar(iosysid, num_flavors, flavor, my_rank, async, test_comm))) + return ret; return PIO_NOERR; } @@ -1880,6 +2133,6 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te int main(int argc, char **argv) { /* Change the 5th arg to 3 to turn on logging. */ - return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0, + return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 3, TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS); } diff --git a/src/externals/pio2/tests/cunit/test_pioc_fill.c b/src/externals/pio2/tests/cunit/test_pioc_fill.c index e2015dee7f0f..c699a566be50 100644 --- a/src/externals/pio2/tests/cunit/test_pioc_fill.c +++ b/src/externals/pio2/tests/cunit/test_pioc_fill.c @@ -440,14 +440,18 @@ int create_putget_file(int iosysid, int flavor, int *dim_len, int *varid, const int xtype[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING}; int ncid; + int old_mode; int ret; /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER))) return ret; + /* This should not work. */ + if (PIOc_set_fill(ncid + TEST_VAL_42, NC_FILL, &old_mode) != PIO_EBADID) + return ret; + /* Turn on fill mode. */ - int old_mode; if ((ret = PIOc_set_fill(ncid, NC_FILL, &old_mode))) return ret; printf("old_mode = %d\n", old_mode); diff --git a/src/externals/pio2/tests/cunit/test_rearr.c b/src/externals/pio2/tests/cunit/test_rearr.c new file mode 100644 index 000000000000..89e278bfc140 --- /dev/null +++ b/src/externals/pio2/tests/cunit/test_rearr.c @@ -0,0 +1,1542 @@ +/* + * This program tests some internal functions in the library related + * to the box and subset rearranger, and the transfer of data betweeen + * IO and computation tasks. + * + * Ed Hartnett, 3/9/17 + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 1 + +/* The name of this test. */ +#define TEST_NAME "test_rearr" + +/* For 1-D use. */ +#define NDIM1 1 + +/* For maplens of 2. */ +#define MAPLEN2 2 + +/* Name of test var. (Name of a Welsh town.)*/ +#define VAR_NAME "Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch" + +/* Test some of the rearranger utility functions. */ +int test_rearranger_opts1(int iosysid) +{ + iosystem_desc_t *ios; + int ret; + + /* This should not work. */ + if (PIOc_set_rearr_opts(TEST_VAL_42, 0, 0, false, false, 0, false, + false, 0) != PIO_EBADID) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, TEST_VAL_42, 0, false, false, 0, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, TEST_VAL_42, false, false, 0, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false, + PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1, false, + false, 0) != PIO_EINVAL) + return ERR_WRONG; + if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false, 0, false, + false, PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1) != + PIO_EINVAL) + return ERR_WRONG; + + /* This should work. */ + if ((ret = PIOc_set_rearr_opts(iosysid, PIO_REARR_COMM_P2P, + PIO_REARR_COMM_FC_1D_COMP2IO, true, + true, TEST_VAL_42, true, true, TEST_VAL_42 + 1))) + return ret; + + /* Get the IO system info from the id. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* Check the rearranger settings. */ + if (ios->rearr_opts.comm_type != PIO_REARR_COMM_P2P || + ios->rearr_opts.fcd != PIO_REARR_COMM_FC_1D_COMP2IO || + !ios->rearr_opts.comp2io.hs || !ios->rearr_opts.comp2io.isend || + !ios->rearr_opts.io2comp.hs || !ios->rearr_opts.io2comp.isend || + ios->rearr_opts.comp2io.max_pend_req != TEST_VAL_42 || + ios->rearr_opts.io2comp.max_pend_req != TEST_VAL_42 + 1) + return ERR_WRONG; + + return 0; +} + +/* Test the compare_offsets() function. */ +int test_compare_offsets() +{ + mapsort m1, m2, m3; + + m1.rfrom = 0; + m1.soffset = 0; + m1.iomap = 0; + m2.rfrom = 0; + m2.soffset = 0; + m2.iomap = 0; + m3.rfrom = 0; + m3.soffset = 0; + m3.iomap = 1; + + /* Return 0 if either or both parameters are null. */ + if (compare_offsets(NULL, &m2)) + return ERR_WRONG; + if (compare_offsets(&m1, NULL)) + return ERR_WRONG; + if (compare_offsets(NULL, NULL)) + return ERR_WRONG; + + /* m1 and m2 are the same. */ + if (compare_offsets(&m1, &m2)) + return ERR_WRONG; + + /* m1 and m3 are different. */ + if (compare_offsets(&m1, &m3) != -1) + return ERR_WRONG; + return 0; +} + +/* Test the ceil2() and pair() functions. */ +int test_ceil2_pair() +{ + /* Test the ceil2() function. */ + if (ceil2(1) != 1) + return ERR_WRONG; + if (ceil2(-100) != 1) + return ERR_WRONG; + if (ceil2(2) != 2) + return ERR_WRONG; + if (ceil2(3) != 4) + return ERR_WRONG; + if (ceil2(16) != 16) + return ERR_WRONG; + if (ceil2(17) != 32) + return ERR_WRONG; + + /* Test the pair() function. */ + if (pair(4, 0, 0) != 1) + return ERR_WRONG; + if (pair(4, 2, 2) != 1) + return ERR_WRONG; + + return 0; +} + +/* Test the create_mpi_datatypes() function. + * @returns 0 for success, error code otherwise.*/ +int test_create_mpi_datatypes() +{ + MPI_Datatype basetype = MPI_INT; + int *mfrom = NULL; + int mpierr; + int ret; + + { + int msgcnt = 1; + PIO_Offset mindex[1] = {0}; + int mcount[1] = {1}; + MPI_Datatype mtype; + + /* Create an MPI data type. */ + if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, &mtype))) + return ret; + + /* Free the type. */ + if ((mpierr = MPI_Type_free(&mtype))) + MPIERR(mpierr); + } + + { + int msgcnt = 4; + PIO_Offset mindex[4] = {0, 0, 0, 0}; + int mcount[4] = {1, 1, 1, 1}; + MPI_Datatype mtype2[4]; + + /* Create 4 MPI data types. */ + if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, mtype2))) + return ret; + + /* Check the size of the data types. It should be 4. */ + MPI_Aint lb, extent; + for (int t = 0; t < 4; t++) + { + if ((mpierr = MPI_Type_get_extent(mtype2[t], &lb, &extent))) + MPIERR(mpierr); + printf("t = %d lb = %ld extent = %ld\n", t, lb, extent); + if (lb != 0 || extent != 4) + return ERR_WRONG; + } + + /* Free them. */ + for (int t = 0; t < 4; t++) + if ((mpierr = MPI_Type_free(&mtype2[t]))) + return ERR_WRONG; + } + + return 0; +} + +/* Test the idx_to_dim_list() function. */ +int test_idx_to_dim_list() +{ + int ndims = 1; + int gdims[1] = {1}; + PIO_Offset idx = 0; + PIO_Offset dim_list[1]; + + /* This simplest case. */ + idx_to_dim_list(ndims, gdims, idx, dim_list); + + if (dim_list[0] != 0) + return ERR_WRONG; + + /* The case given in the function docs. */ + int ndims2 = 2; + int gdims2[2] = {3, 2}; + PIO_Offset idx2 = 4; + PIO_Offset dim_list2[2]; + + /* According to function docs, we should get 2,0 */ + idx_to_dim_list(ndims2, gdims2, idx2, dim_list2); + printf("dim_list2[0] = %lld\n", dim_list2[0]); + printf("dim_list2[1] = %lld\n", dim_list2[1]); + + /* This is the correct result! */ + if (dim_list2[0] != 2 || dim_list2[1] != 0) + return ERR_WRONG; + + return 0; +} + +/* Test the coord_to_lindex() function. */ +int test_coord_to_lindex() +{ + int ndims = 1; + PIO_Offset lcoord[1] = {0}; + PIO_Offset count[1] = {1}; + PIO_Offset lindex; + + /* Not sure what this function is really doing. */ + lindex = coord_to_lindex(ndims, lcoord, count); + if (lindex != 0) + return ERR_WRONG; + + int ndims2 = 2; + PIO_Offset lcoord2[2] = {0, 0}; + PIO_Offset count2[2] = {1, 1}; + PIO_Offset lindex2; + + lindex2 = coord_to_lindex(ndims2, lcoord2, count2); + if (lindex2 != 0) + return ERR_WRONG; + + int ndims3 = 2; + PIO_Offset lcoord3[2] = {1, 2}; + PIO_Offset count3[2] = {1, 1}; + PIO_Offset lindex3; + + lindex3 = coord_to_lindex(ndims3, lcoord3, count3); + printf("lindex = %lld\n", lindex3); + if (lindex3 != 3) + return ERR_WRONG; + + return 0; +} + +/* Test compute_maxIObuffersize() function. */ +int test_compute_maxIObuffersize(MPI_Comm test_comm, int my_rank) +{ + int ret; + + { + /* This is a simple test with one region containing 1 data + * element. */ + io_desc_t iodesc; + io_region *ior1; + int ndims = 1; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior1))) + return ret; + ior1->next = NULL; + ior1->count[0] = 1; + + iodesc.firstregion = ior1; + iodesc.ndims = 1; + + /* Run the function. Simplest possible case. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + if (iodesc.maxiobuflen != 1) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior1->start); + free(ior1->count); + free(ior1); + + } + + { + /* This also has a single region, but with 2 dims and count + * values > 1. */ + io_desc_t iodesc; + io_region *ior2; + int ndims = 2; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior2))) + return ret; + + /* These should be 0. */ + for (int i = 0; i < ndims; i++) + if (ior2->start[i] != 0 || ior2->count[i] != 0) + return ERR_WRONG; + + ior2->next = NULL; + ior2->count[0] = 10; + ior2->count[1] = 2; + + iodesc.firstregion = ior2; + iodesc.ndims = 2; + + /* Run the function. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + if (iodesc.maxiobuflen != 20) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior2->start); + free(ior2->count); + free(ior2); + } + + { + /* This test has two regions of different sizes. */ + io_desc_t iodesc; + io_region *ior3; + io_region *ior4; + int ndims = 2; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, ndims, &ior4))) + return ret; + ior4->next = NULL; + ior4->count[0] = 10; + ior4->count[1] = 2; + + if ((ret = alloc_region2(NULL, ndims, &ior3))) + return ret; + ior3->next = ior4; + ior3->count[0] = 100; + ior3->count[1] = 5; + + iodesc.firstregion = ior3; + iodesc.ndims = 2; + + /* Run the function. */ + if ((ret = compute_maxIObuffersize(test_comm, &iodesc))) + return ret; + printf("iodesc.maxiobuflen = %d\n", iodesc.maxiobuflen); + if (iodesc.maxiobuflen != 520) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior4->start); + free(ior4->count); + free(ior4); + free(ior3->start); + free(ior3->count); + free(ior3); + } + + return 0; +} + +/* Tests for determine_fill() function. */ +int test_determine_fill(MPI_Comm test_comm) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int gsize[1] = {4}; + PIO_Offset compmap[1] = {1}; + int ret; + + /* Initialize ios. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + ios->union_comm = test_comm; + + /* Set up iodesc for test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + iodesc->ndims = 1; + iodesc->rearranger = PIO_REARR_SUBSET; + iodesc->llen = 1; + + /* We don't need fill. */ + if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + return ret; + if (iodesc->needsfill) + return ERR_WRONG; + + /* Change settings, so now we do need fill. */ + iodesc->llen = 0; + if ((ret = determine_fill(ios, iodesc, gsize, compmap))) + return ret; + if (!iodesc->needsfill) + return ERR_WRONG; + + /* Free test resources. */ + free(ios); + free(iodesc); + + return 0; +} + +/* Run tests for get_start_and_count_regions() funciton. */ +int test_get_regions(int my_rank) +{ +#define MAPLEN 2 + int ndims = NDIM1; + const int gdimlen[NDIM1] = {8}; + /* Don't forget map is 1-based!! */ + PIO_Offset map[MAPLEN] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1}; + int maxregions; + io_region *ior1; + int ret; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + ior1->count[0] = 1; + + /* Call the function we are testing. */ + if ((ret = get_regions(ndims, gdimlen, MAPLEN, map, &maxregions, ior1))) + return ret; + if (maxregions != 2) + return ERR_WRONG; + + /* Free resources for the region. */ + free(ior1->next->start); + free(ior1->next->count); + free(ior1->next); + free(ior1->start); + free(ior1->count); + free(ior1); + + return 0; +} + +/* Run tests for find_region() function. */ +int test_find_region() +{ + int ndims = NDIM1; + int gdimlen[NDIM1] = {4}; + int maplen = 1; + PIO_Offset map[1] = {1}; + PIO_Offset start[NDIM1]; + PIO_Offset count[NDIM1]; + PIO_Offset regionlen; + + /* Call the function we are testing. */ + regionlen = find_region(ndims, gdimlen, maplen, map, start, count); + + /* Check results. */ + printf("regionlen = %lld start[0] = %lld count[0] = %lld\n", regionlen, start[0], count[0]); + if (regionlen != 1 || start[0] != 0 || count[0] != 1) + return ERR_WRONG; + + return 0; +} + +/* Run tests for expand_region() function. */ +int test_expand_region() +{ + int dim = 0; + int gdims[NDIM1] = {1}; + int maplen = 1; + PIO_Offset map[1] = {5}; + int region_size = 1; + int region_stride = 1; + int max_size[NDIM1] = {10}; + PIO_Offset count[NDIM1]; + + expand_region(dim, gdims, maplen, map, region_size, region_stride, max_size, count); + if (count[0] != 1) + return ERR_WRONG; + printf("max_size[0] = %d count[0] = %lld\n", max_size[0], count[0]); + + return 0; +} + +/* Test define_iodesc_datatypes() function. */ +int test_define_iodesc_datatypes() +{ +#define NUM_REARRANGERS 2 + int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int mpierr; + int ret; + + /* Run the functon. */ + for (int r = 0; r < NUM_REARRANGERS; r++) + { + iosystem_desc_t ios; + io_desc_t iodesc; + + /* Set up test for IO task with BOX rearranger to create one type. */ + ios.ioproc = 1; /* this is IO proc. */ + ios.num_iotasks = 4; /* The number of IO tasks. */ + iodesc.rtype = NULL; /* Array of MPI types will be created here. */ + iodesc.nrecvs = 1; /* Number of types created. */ + iodesc.basetype = MPI_INT; + iodesc.stype = NULL; /* Array of MPI types will be created here. */ + + /* Allocate space for arrays in iodesc that will be filled in + * define_iodesc_datatypes(). */ + if (!(iodesc.rcount = malloc(iodesc.nrecvs * sizeof(int)))) + return PIO_ENOMEM; + if (!(iodesc.rfrom = malloc(iodesc.nrecvs * sizeof(int)))) + return PIO_ENOMEM; + if (!(iodesc.rindex = malloc(1 * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + iodesc.rindex[0] = 0; + iodesc.rcount[0] = 1; + + iodesc.rearranger = rearranger[r]; + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc.rearranger == PIO_REARR_BOX ? ios.num_iotasks : 1; + + if (!(iodesc.sindex = malloc(num_send_types * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + if (!(iodesc.scount = malloc(num_send_types * sizeof(int)))) + return PIO_ENOMEM; + for (int st = 0; st < num_send_types; st++) + { + iodesc.sindex[st] = 0; + iodesc.scount[st] = 1; + } + + /* Run the test function. */ + if ((ret = define_iodesc_datatypes(&ios, &iodesc))) + return ret; + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if ((mpierr = MPI_Type_free(&iodesc.stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if ((mpierr = MPI_Type_free(&iodesc.rtype[0]))) + MPIERR(mpierr); + + /* Free resources. */ + free(iodesc.rtype); + free(iodesc.sindex); + free(iodesc.scount); + free(iodesc.stype); + free(iodesc.rcount); + free(iodesc.rfrom); + free(iodesc.rindex); + } + + return 0; +} + +/* Test the compute_counts() function with the box rearranger. */ +int test_compute_counts(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int dest_ioproc[TARGET_NTASKS] = {0, 1, 2, 3}; + PIO_Offset dest_ioindex[TARGET_NTASKS] = {0, 1, 2, 3}; + int ret; + + /* Initialize ios. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + ios->num_iotasks = TARGET_NTASKS; + ios->num_comptasks = TARGET_NTASKS; + ios->num_uniontasks = TARGET_NTASKS; + ios->ioproc = 1; + ios->compproc = 1; + ios->union_comm = test_comm; + if (!(ios->ioranks = malloc(TARGET_NTASKS * sizeof(int)))) + return PIO_ENOMEM; + for (int t = 0; t < TARGET_NTASKS; t++) + ios->ioranks[t] = t; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* Initialize iodesc. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->ndof = TARGET_NTASKS; + iodesc->llen = TARGET_NTASKS; + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Test the function. */ + if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex))) + return ret; + + /* Check results. */ + for (int i = 0; i < ios->num_iotasks; i++) + if (iodesc->scount[i] != 1 || iodesc->sindex[i] != i) + return ERR_WRONG; + + for (int i = 0; i < iodesc->ndof; i++) + if (iodesc->rcount[i] != 1 || iodesc->rfrom[i] != i || + iodesc->rindex[i] != my_rank) + return ERR_WRONG; + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free test resources. */ + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Call PIOc_InitDecomp() with parameters such that it calls + * box_rearrange_create() just like test_box_rearrange_create() will + * (see below). */ +int test_init_decomp(int iosysid, MPI_Comm test_comm, int my_rank) +{ + int ioid; + PIO_Offset compmap[MAPLEN2] = {my_rank * 2, (my_rank + 1) * 2}; + const int gdimlen[NDIM1] = {8}; + int ret; + + /* Initialize a decomposition. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN2, + compmap, &ioid, PIO_REARR_BOX, NULL, NULL))) + return ret; + + /* Free it. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + return ret; + + return 0; +} + +/* Test for the box_rearrange_create() function. */ +int test_box_rearrange_create(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + io_region *ior1; + int maplen = MAPLEN2; + PIO_Offset compmap[MAPLEN2] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* We are finally ready to run the code under test. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Check some results. */ + if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen || + iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill) + return ERR_WRONG; + + /* for (int i = 0; i < ios->num_iotasks; i++) */ + /* { */ + /* /\* sindex is only allocated if scount[i] > 0. *\/ */ + /* if (iodesc->scount[i] != i ? 0 : 1 || */ + /* (iodesc->scount[i] && iodesc->sindex[i] != 0)) */ + /* return ERR_WRONG; */ + /* } */ + + /* for (int i = 0; i < iodesc->ndof; i++) */ + /* { */ + /* /\* rcount is 1 for rank 0, 0 on other tasks. *\/ */ + /* if (iodesc->rcount[i] != my_rank ? 0 : 1) */ + /* return ERR_WRONG; */ + + /* /\* rfrom is 0 everywhere, except task 0, array elemnt 1. *\/ */ + /* if (my_rank == 0 && i == 1) */ + /* { */ + /* if (iodesc->rfrom[i] != 1) */ + /* return ERR_WRONG; */ + /* } */ + /* else */ + /* { */ + /* if (iodesc->rfrom[i] != 0) */ + /* return ERR_WRONG; */ + /* } */ + + /* /\* rindex is only allocated where there is a non-zero count. *\/ */ + /* if (iodesc->rcount[i]) */ + /* if (iodesc->rindex[i] != 0) */ + /* return ERR_WRONG; */ + /* } */ + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Test for the box_rearrange_create() function. */ +int test_box_rearrange_create_2(MPI_Comm test_comm, int my_rank) +{ +#define MAPLEN2 2 + iosystem_desc_t *ios; + io_desc_t *iodesc; + io_region *ior1; + int maplen = MAPLEN2; + PIO_Offset compmap[MAPLEN2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + /* This is the size of the map in computation tasks. */ + iodesc->ndof = 2; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* We are finally ready to run the code under test. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Check some results. */ + if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen || + iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill) + return ERR_WRONG; + + for (int i = 0; i < ios->num_iotasks; i++) + { + /* sindex is only allocated if scount[i] > 0. */ + if (iodesc->scount[i] != i ? 0 : 1 || + (iodesc->scount[i] && iodesc->sindex[i] != 0)) + return ERR_WRONG; + } + + if (my_rank == 0) + { + for (int i = 0; i < iodesc->ndof; i++) + { + /* rcount is 1 for rank 0, 0 on other tasks. */ + if (iodesc->rcount[i] != 1) + return ERR_WRONG; + + /* rfrom only matters if there is a non-zero count. */ + if (iodesc->rcount[i]) + if (iodesc->rfrom[i] != i ? 1 : 0) + return ERR_WRONG; + + /* rindex is only allocated where there is a non-zero count. */ + if (iodesc->rcount[i]) + if (iodesc->rindex[i] != 0) + return ERR_WRONG; + } + } + + /* Free resources allocated in compute_counts(). */ + free(iodesc->scount); + free(iodesc->sindex); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + + return 0; +} + +/* Test function default_subset_partition. */ +int test_default_subset_partition(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + int mpierr; + int ret; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->io_rank = my_rank; + ios->comp_comm = test_comm; + + /* Run the function to test. */ + if ((ret = default_subset_partition(ios, iodesc))) + return ret; + + /* Free the created communicator. */ + if ((mpierr = MPI_Comm_free(&iodesc->subset_comm))) + MPIERR(mpierr); + + /* Free resources from test. */ + free(iodesc); + free(ios); + + return 0; +} + +/* Test function rearrange_comp2io. */ +int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + void *sbuf = NULL; + void *rbuf = NULL; + int nvars = 1; + io_region *ior1; + int maplen = 2; + PIO_Offset compmap[2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int mpierr; + int ret; + + /* Allocate some space for data. */ + if (!(sbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + if (!(rbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->compproc = 1; + ios->io_rank = my_rank; + ios->union_comm = test_comm; + ios->num_iotasks = TARGET_NTASKS; + ios->num_uniontasks = TARGET_NTASKS; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->basetype = MPI_INT; + + /* Set up test for IO task with BOX rearranger to create one type. */ + iodesc->rtype = NULL; /* Array of MPI types will be created here. */ + iodesc->nrecvs = 1; /* Number of types created. */ + iodesc->basetype = MPI_INT; + iodesc->stype = NULL; /* Array of MPI types will be created here. */ + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + iodesc->ndof = 4; + + /* Set up the IO task info for the test. */ + ios->union_rank = my_rank; + ios->num_comptasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* Create the box rearranger. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Run the function to test. */ + if ((ret = rearrange_comp2io(ios, iodesc, sbuf, rbuf, nvars))) + return ret; + printf("returned from rearrange_comp2io\n"); + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if (iodesc->stype[st] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if (iodesc->rtype) + for (int r = 0; r < iodesc->nrecvs; r++) + if (iodesc->rtype[r] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->rtype[r]))) + MPIERR(mpierr); + + /* Free resources allocated in library code. */ + free(iodesc->rtype); + free(iodesc->sindex); + free(iodesc->scount); + free(iodesc->stype); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + free(sbuf); + free(rbuf); + + return 0; +} + +/* Test function rearrange_io2comp. */ +int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank) +{ + iosystem_desc_t *ios; + io_desc_t *iodesc; + void *sbuf = NULL; + void *rbuf = NULL; + io_region *ior1; + int maplen = 2; + PIO_Offset compmap[2] = {1, 0}; + const int gdimlen[NDIM1] = {8}; + int ndims = NDIM1; + int mpierr; + int ret; + + /* Allocate some space for data. */ + if (!(sbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + if (!(rbuf = calloc(4, sizeof(int)))) + return PIO_ENOMEM; + + /* Allocate IO system info struct for this test. */ + if (!(ios = calloc(1, sizeof(iosystem_desc_t)))) + return PIO_ENOMEM; + + /* Allocate IO desc struct for this test. */ + if (!(iodesc = calloc(1, sizeof(io_desc_t)))) + return PIO_ENOMEM; + + ios->ioproc = 1; + ios->io_rank = my_rank; + ios->union_comm = test_comm; + ios->num_iotasks = TARGET_NTASKS; + iodesc->rearranger = PIO_REARR_BOX; + iodesc->basetype = MPI_INT; + + /* Set up test for IO task with BOX rearranger to create one type. */ + ios->ioproc = 1; /* this is IO proc. */ + ios->num_iotasks = 4; /* The number of IO tasks. */ + iodesc->rtype = NULL; /* Array of MPI types will be created here. */ + iodesc->nrecvs = 1; /* Number of types created. */ + iodesc->basetype = MPI_INT; + iodesc->stype = NULL; /* Array of MPI types will be created here. */ + + /* The two rearrangers create a different number of send types. */ + int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1; + + /* Default rearranger options. */ + iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL; + iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE; + + /* Set up for determine_fill(). */ + ios->union_comm = test_comm; + ios->io_comm = test_comm; + iodesc->ndims = NDIM1; + iodesc->rearranger = PIO_REARR_BOX; + + iodesc->ndof = 4; + + /* Set up the IO task info for the test. */ + ios->ioproc = 1; + ios->compproc = 1; + ios->union_rank = my_rank; + ios->num_iotasks = 4; + ios->num_comptasks = 4; + ios->num_uniontasks = 4; + if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->ioranks[i] = i; + if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int i = 0; i < TARGET_NTASKS; i++) + ios->compranks[i] = i; + + /* This is how we allocate a region. */ + if ((ret = alloc_region2(NULL, NDIM1, &ior1))) + return ret; + ior1->next = NULL; + if (my_rank == 0) + ior1->count[0] = 8; + + iodesc->firstregion = ior1; + + /* Create the box rearranger. */ + if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + return ret; + + /* Run the function to test. */ + if ((ret = rearrange_io2comp(ios, iodesc, sbuf, rbuf))) + return ret; + printf("returned from rearrange_comp2io\n"); + + /* We created send types, so free them. */ + for (int st = 0; st < num_send_types; st++) + if (iodesc->stype[st] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->stype[st]))) + MPIERR(mpierr); + + /* We created one receive type, so free it. */ + if (iodesc->rtype) + for (int r = 0; r < iodesc->nrecvs; r++) + if (iodesc->rtype[r] != PIO_DATATYPE_NULL) + if ((mpierr = MPI_Type_free(&iodesc->rtype[r]))) + MPIERR(mpierr); + + /* Free resources allocated in library code. */ + free(iodesc->rtype); + free(iodesc->sindex); + free(iodesc->scount); + free(iodesc->stype); + free(iodesc->rcount); + free(iodesc->rfrom); + free(iodesc->rindex); + + /* Free resources from test. */ + free(ior1->start); + free(ior1->count); + free(ior1); + free(ios->ioranks); + free(ios->compranks); + free(iodesc); + free(ios); + free(sbuf); + free(rbuf); + + return 0; +} + +/* These tests do not need an iosysid. */ +int run_no_iosys_tests(int my_rank, MPI_Comm test_comm) +{ + int ret; + + printf("%d running idx_to_dim_list tests\n", my_rank); + if ((ret = test_idx_to_dim_list())) + return ret; + + printf("%d running coord_to_lindex tests\n", my_rank); + if ((ret = test_coord_to_lindex())) + return ret; + + printf("%d running compute_maxIObuffersize tests\n", my_rank); + if ((ret = test_compute_maxIObuffersize(test_comm, my_rank))) + return ret; + + printf("%d running determine_fill\n", my_rank); + if ((ret = test_determine_fill(test_comm))) + return ret; + + printf("%d running tests for expand_region()\n", my_rank); + if ((ret = test_expand_region())) + return ret; + + printf("%d running tests for find_region()\n", my_rank); + if ((ret = test_find_region())) + return ret; + + printf("%d running tests for get_regions()\n", my_rank); + if ((ret = test_get_regions(my_rank))) + return ret; + + printf("%d running create_mpi_datatypes tests\n", my_rank); + if ((ret = test_create_mpi_datatypes())) + return ret; + + printf("%d running define_iodesc_datatypes tests\n", my_rank); + if ((ret = test_define_iodesc_datatypes())) + return ret; + + printf("%d running compare_offsets tests\n", my_rank); + if ((ret = test_compare_offsets())) + return ret; + + printf("%d running compute_counts tests for box rearranger\n", my_rank); + if ((ret = test_compute_counts(test_comm, my_rank))) + return ret; + + printf("%d running tests for box_rearrange_create\n", my_rank); + if ((ret = test_box_rearrange_create(test_comm, my_rank))) + return ret; + + printf("%d running more tests for box_rearrange_create\n", my_rank); + if ((ret = test_box_rearrange_create_2(test_comm, my_rank))) + return ret; + + printf("%d running tests for default_subset_partition\n", my_rank); + if ((ret = test_default_subset_partition(test_comm, my_rank))) + return ret; + + printf("%d running tests for rearrange_comp2io\n", my_rank); + if ((ret = test_rearrange_comp2io(test_comm, my_rank))) + return ret; + + printf("%d running tests for rearrange_io2comp\n", my_rank); + if ((ret = test_rearrange_io2comp(test_comm, my_rank))) + return ret; + + return 0; +} + +/* Test scalar vars. */ +int test_scalar(int numio, int iosysid, MPI_Comm test_comm, int my_rank, + int num_flavors, int *flavor) +{ + + int var_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, + PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, + PIO_UINT64}; + char char_data = 2; + signed char byte_data = -42; + short short_data = -300; + int int_data = -10000; + float float_data = -42.42; + double double_data = -420000000000.5; + unsigned char ubyte_data = 43; + unsigned short ushort_data = 666; + unsigned int uint_data = 666666; + long long int64_data = -99999999999; + unsigned long long uint64_data = 99999999999; + char char_data_in; + signed char byte_data_in; + short short_data_in; + unsigned char ubyte_data_in; + int int_data_in; + float float_data_in; + double double_data_in; + unsigned short ushort_data_in; + unsigned int uint_data_in; + long long int64_data_in; + unsigned long long uint64_data_in; + + int ret; + + /* Run tests with all available iotypes. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + /* For netcdf-4, there are extra types. */ + int num_types = (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) ? + NUM_NETCDF4_TYPES - 1 : NUM_CLASSIC_TYPES; + + /* For each available type, create a file with a scalar var of + * that type. */ + for (int t = 0; t < num_types; t++) + { + int ncid; + int varid; + char filename[PIO_MAX_NAME + 1]; + + printf("test with t = %d\n", t); + + /* These iotypes only handle netCDF classic types. */ + if (t >= NUM_CLASSIC_TYPES && + (flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF)) + continue; + + /* Create filename. */ + sprintf(filename, "%s_scalar_numio_%d_iotype_%d_var_type_%d.nc", TEST_NAME, + numio, flavor[fmt], var_type[t]); + + /* Create the file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER))) + return ret; + + /* Define scalar var. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, var_type[t], 0, NULL, &varid))) + return ret; + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + return ret; + + /* Write a value. */ + switch (var_type[t]) + { + case PIO_BYTE: + if ((ret = PIOc_put_var_schar(ncid, varid, &byte_data))) + return ret; + break; + case PIO_CHAR: + if ((ret = PIOc_put_var_text(ncid, varid, &char_data))) + return ret; + break; + case PIO_SHORT: + if ((ret = PIOc_put_var_short(ncid, varid, &short_data))) + return ret; + break; + case PIO_INT: + if ((ret = PIOc_put_var_int(ncid, varid, &int_data))) + return ret; + break; + case PIO_FLOAT: + if ((ret = PIOc_put_var_float(ncid, varid, &float_data))) + return ret; + break; + case PIO_DOUBLE: + if ((ret = PIOc_put_var_double(ncid, varid, &double_data))) + return ret; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if ((ret = PIOc_put_var_uchar(ncid, varid, &ubyte_data))) + return ret; + break; + case PIO_USHORT: + if ((ret = PIOc_put_var_ushort(ncid, varid, &ushort_data))) + return ret; + break; + case PIO_UINT: + if ((ret = PIOc_put_var_uint(ncid, varid, &uint_data))) + return ret; + break; + case PIO_INT64: + if ((ret = PIOc_put_var_longlong(ncid, varid, &int64_data))) + return ret; + break; + case PIO_UINT64: + if ((ret = PIOc_put_var_ulonglong(ncid, varid, &uint64_data))) + return ret; + break; +#endif /* _NETCDF4 */ + default: + return ERR_WRONG; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, NC_NOWRITE))) + return ret; + + /* Read the value. Is the value correct? */ + switch (var_type[t]) + { + case PIO_BYTE: + if ((ret = PIOc_get_var_schar(ncid, varid, &byte_data_in))) + return ret; + if (byte_data_in != byte_data) + return ERR_WRONG; + break; + case PIO_CHAR: + if ((ret = PIOc_get_var_text(ncid, varid, &char_data_in))) + return ret; + if (char_data_in != char_data) + return ERR_WRONG; + break; + case PIO_SHORT: + if ((ret = PIOc_get_var_short(ncid, varid, &short_data_in))) + return ret; + if (short_data_in != short_data) + return ERR_WRONG; + break; + case PIO_INT: + if ((ret = PIOc_get_var_int(ncid, varid, &int_data_in))) + return ret; + if (int_data_in != int_data) + return ERR_WRONG; + break; + case PIO_FLOAT: + if ((ret = PIOc_get_var_float(ncid, varid, &float_data_in))) + return ret; + if (float_data_in != float_data) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if ((ret = PIOc_get_var_double(ncid, varid, &double_data_in))) + return ret; + if (double_data_in != double_data) + return ERR_WRONG; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if ((ret = PIOc_get_var_uchar(ncid, varid, &ubyte_data_in))) + return ret; + if (ubyte_data_in != ubyte_data) + return ERR_WRONG; + break; + case PIO_USHORT: + if ((ret = PIOc_get_var_ushort(ncid, varid, &ushort_data_in))) + return ret; + if (ushort_data_in != ushort_data) + return ERR_WRONG; + break; + case PIO_UINT: + if ((ret = PIOc_get_var_uint(ncid, varid, &uint_data_in))) + return ret; + if (uint_data_in != uint_data) + return ERR_WRONG; + break; + case PIO_INT64: + if ((ret = PIOc_get_var_longlong(ncid, varid, &int64_data_in))) + return ret; + if (int64_data_in != int64_data) + return ERR_WRONG; + break; + case PIO_UINT64: + if ((ret = PIOc_get_var_ulonglong(ncid, varid, &uint64_data_in))) + return ret; + if (uint64_data_in != uint64_data) + return ERR_WRONG; + break; +#endif /* _NETCDF4 */ + default: + return ERR_WRONG; + } + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + } /* next iotype */ + } /* next type */ + + return 0; +} + +/* These tests are run with different rearrangers and numbers of IO + * tasks. */ +int run_iosys_tests(int numio, int iosysid, int my_rank, MPI_Comm test_comm, + int num_flavors, int *flavor) +{ + int ret; + + printf("%d running rearranger opts tests 1\n", my_rank); + if ((ret = test_rearranger_opts1(iosysid))) + return ret; + + printf("%d running test for init_decomp\n", my_rank); + if ((ret = test_init_decomp(iosysid, test_comm, my_rank))) + return ret; + + printf("%d running test for init_decomp\n", my_rank); + if ((ret = test_scalar(numio, iosysid, test_comm, my_rank, num_flavors, flavor))) + return ret; + + return 0; +} + +/* Run Tests for pio_spmd.c functions. */ +int main(int argc, char **argv) +{ + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + TARGET_NTASKS, 3, &test_comm))) + ERR(ERR_INIT); + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + printf("Runnings tests for %d flavors\n", num_flavors); + + /* Test code runs on TARGET_NTASKS tasks. The left over tasks do + * nothing. */ + if (my_rank < TARGET_NTASKS) + { + /* Run the tests that don't need an iosysid. */ + if ((ret = run_no_iosys_tests(my_rank, test_comm))) + return ret; + + /* Test code with both rearrangers. */ + for (int r = 0; r < NUM_REARRANGERS; r++) + { + /* Test code with 1, 2, 3, and 4 io tasks. */ + for (int numio = 1; numio <= TARGET_NTASKS; numio++) + { + int iosysid; + int rearranger = r ? PIO_REARR_SUBSET : PIO_REARR_BOX; + + if ((ret = PIOc_Init_Intracomm(test_comm, numio, 1, 0, rearranger, + &iosysid))) + return ret; + + /* Run the tests that need an iosysid. */ + if ((ret = run_iosys_tests(numio, iosysid, my_rank, test_comm, + num_flavors, flavor))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* next numio */ + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + printf("%d %s Finalizing...\n", my_rank, TEST_NAME); + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + + return 0; +} diff --git a/src/externals/pio2/tests/cunit/test_shared.c b/src/externals/pio2/tests/cunit/test_shared.c index 3d6eb9cd1ffb..0926e0be9fd9 100644 --- a/src/externals/pio2/tests/cunit/test_shared.c +++ b/src/externals/pio2/tests/cunit/test_shared.c @@ -20,22 +20,21 @@ int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int component_count, int num_io_procs, int target_ntasks, char *test_name) { int iosysid[component_count]; /* The ID for the parallel I/O system. */ - int num_procs[component_count + 1]; /* Num procs in each component. */ + int num_procs[component_count]; /* Num procs in each component. */ MPI_Comm io_comm; /* Will get a duplicate of IO communicator. */ MPI_Comm comp_comm[component_count]; /* Will get duplicates of computation communicators. */ int mpierr; /* Return code from MPI functions. */ int ret; /* Return code. */ - num_procs[0] = 1; - num_procs[1] = target_ntasks - 1; + num_procs[0] = target_ntasks - 1; /* Is the current process a computation task? */ int comp_task = my_rank < num_io_procs ? 0 : 1; printf("%d comp_task = %d\n", my_rank, comp_task); /* Initialize the IO system. */ - if ((ret = PIOc_Init_Async(test_comm, num_io_procs, NULL, component_count, - num_procs, NULL, &io_comm, comp_comm, iosysid))) + if ((ret = PIOc_init_async(test_comm, num_io_procs, NULL, component_count, + num_procs, NULL, &io_comm, comp_comm, PIO_REARR_BOX, iosysid))) ERR(ERR_INIT); for (int c = 0; c < component_count; c++) printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]); @@ -103,17 +102,17 @@ int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm ioproc_start, PIO_REARR_SUBSET, &iosysid))) return ret; - /* Describe the decomposition. This is a 1-based array, so add 1! */ + /* Describe the decomposition. This is a 0-based array, so don't add 1! */ elements_per_pe = x_dim_len * y_dim_len / target_ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i + 1; + compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this test. */ printf("%d Creating decomposition...\n", my_rank); - if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, - compdof, &ioid, NULL, NULL, NULL))) + if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, + compdof, &ioid, 0, NULL, NULL))) return ret; free(compdof); diff --git a/src/externals/pio2/tests/cunit/test_spmd.c b/src/externals/pio2/tests/cunit/test_spmd.c index b4947137004f..2c51ce2ccab3 100644 --- a/src/externals/pio2/tests/cunit/test_spmd.c +++ b/src/externals/pio2/tests/cunit/test_spmd.c @@ -1,5 +1,5 @@ /* - * This program tests some internal functions in the library. + * This program tests some internal functions in the PIO library. * * Jim Edwards * Ed Hartnett, 11/23/16 @@ -20,8 +20,6 @@ /* Number of test cases in inner loop of test. */ #define NUM_TEST_CASES 5 -#define TEST_MAX_GATHER_BLOCK_SIZE 32 - /* Test MPI_Alltoallw by having processor i send different amounts of * data to each processor. The first test sends i items to processor * i from all processors. */ @@ -85,8 +83,9 @@ int run_spmd_tests(MPI_Comm test_comm) for (int itest = 0; itest < NUM_TEST_CASES; itest++) { - bool hs = false; - bool isend = false; + rearr_comm_fc_opt_t fc; + fc.hs = false; + fc.isend = false; /* Wait for all tasks. */ MPI_Barrier(test_comm); @@ -99,28 +98,28 @@ int run_spmd_tests(MPI_Comm test_comm) /* Set the parameters different for each test case. */ if (itest == 1) { - hs = true; - isend = true; + fc.hs = true; + fc.isend = true; } else if (itest == 2) { - hs = false; - isend = true; + fc.hs = false; + fc.isend = true; } else if (itest == 3) { - hs = false; - isend = false; + fc.hs = false; + fc.isend = false; } else if (itest == 4) { - hs = true; - isend = false; + fc.hs = true; + fc.isend = false; } /* Run the swapm function. */ if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, - rdispls, recvtypes, test_comm, hs, isend, msg_cnt))) + rdispls, recvtypes, test_comm, &fc))) return ret; /* Print results. */ @@ -189,59 +188,19 @@ int run_sc_tests(MPI_Comm test_comm) if (gcd_array(SC_ARRAY_LEN, array4) != 1) return ERR_WRONG; - return 0; -} - -/* This test code was recovered from main() in pioc_sc.c. */ -int test_CalcStartandCount() -{ - int ndims = 2; - int gdims[2] = {31, 777602}; - int num_io_procs = 24; - bool converged = false; - PIO_Offset start[ndims], kount[ndims]; - int iorank, numaiotasks = 0; - long int tpsize = 0; - long int psize; - long int pgdims = 1; - int scnt; - - for (int i = 0; i < ndims; i++) - pgdims *= gdims[i]; - - while (!converged) - { - for (iorank = 0; iorank < num_io_procs; iorank++) - { - numaiotasks = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank, - start, kount); - if (iorank < numaiotasks) - printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0], - start[1], kount[0], kount[1]); - - if (numaiotasks < 0) - return numaiotasks; - - psize = 1; - scnt = 0; - for (int i = 0; i < ndims; i++) - { - psize *= kount[i]; - scnt += kount[i]; - } - tpsize += psize; - } - - if (tpsize == pgdims) - converged = true; - else - { - printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs); - tpsize = 0; - num_io_procs--; - } - } - + /* Test compute_one_dim. */ + PIO_Offset start, count; + compute_one_dim(4, 4, my_rank, &start, &count); + if (start != my_rank || count != 1) + return ERR_WRONG; + compute_one_dim(400, 4, my_rank, &start, &count); + if (start != my_rank * 100 || count != 100) + return ERR_WRONG; + /* Left over data will go to task 3. */ + compute_one_dim(5, 4, my_rank, &start, &count); + if (start != my_rank || count != (my_rank == 3 ? 2 : 1)) + return ERR_WRONG; + printf("my_rank = %d start = %lld count = %lld\n", my_rank, start, count); return 0; } @@ -264,94 +223,6 @@ int test_lists() return 0; } -/* Test some of the rearranger utility functions. */ -int test_rearranger_opts1() -{ - rearr_comm_fc_opt_t *ro1; - rearr_comm_fc_opt_t *ro2; - rearr_comm_fc_opt_t *ro3; - - if (!(ro1 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - if (!(ro2 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - if (!(ro3 = calloc(1, sizeof(rearr_comm_fc_opt_t)))) - return ERR_AWFUL; - - /* This should not work. */ - if (PIOc_set_rearr_opts(42, 1, 1, 0, 0, 0, 0, 0, 0) != PIO_EBADID) - return ERR_WRONG; - - /* ro1 and ro2 are the same. */ - if (!cmp_rearr_comm_fc_opts(ro1, ro2)) - return ERR_WRONG; - - /* Make ro3 different. */ - ro3->enable_hs = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - ro3->enable_hs = 0; - ro3->enable_isend = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - ro3->enable_isend = 0; - ro3->max_pend_req = 1; - if (cmp_rearr_comm_fc_opts(ro1, ro3)) - return ERR_WRONG; - - /* Free resourses. */ - free(ro1); - free(ro2); - free(ro3); - - return 0; -} - -/* Test some of the rearranger utility functions. */ -int test_rearranger_opts2() -{ - iosystem_desc_t my_ios; - iosystem_desc_t *ios = &my_ios; - - /* I'm not sure what the point of this function is... */ - check_and_reset_rearr_opts(ios); - - return 0; -} - -/* Test the compare_offsets() function. */ -int test_compare_offsets() -{ - mapsort m1, m2, m3; - - m1.rfrom = 0; - m1.soffset = 0; - m1.iomap = 0; - m2.rfrom = 0; - m2.soffset = 0; - m2.iomap = 0; - m3.rfrom = 0; - m3.soffset = 0; - m3.iomap = 1; - - /* Return 0 if either or both parameters are null. */ - if (compare_offsets(NULL, &m2)) - return ERR_WRONG; - if (compare_offsets(&m1, NULL)) - return ERR_WRONG; - if (compare_offsets(NULL, NULL)) - return ERR_WRONG; - - /* m1 and m2 are the same. */ - if (compare_offsets(&m1, &m2)) - return ERR_WRONG; - - /* m1 and m3 are different. */ - if (compare_offsets(&m1, &m3) != -1) - return ERR_WRONG; - return 0; -} - /* Test the ceil2() and pair() functions. */ int test_ceil2_pair() { @@ -382,72 +253,85 @@ int test_ceil2_pair() int test_find_mpi_type() { MPI_Datatype mpi_type; + int type_size; int ret; /* This should not work. */ - if (find_mpi_type(PIO_BYTE + 42, &mpi_type) != PIO_EBADTYPE) + if (find_mpi_type(PIO_BYTE + 42, &mpi_type, &type_size) != PIO_EBADTYPE) return ERR_WRONG; /* Try every atomic type. */ - if ((ret = find_mpi_type(PIO_BYTE, &mpi_type))) + if ((ret = find_mpi_type(PIO_BYTE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_BYTE) + if (mpi_type != MPI_BYTE || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_CHAR, &mpi_type))) + if ((ret = find_mpi_type(PIO_CHAR, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_CHAR) + if (mpi_type != MPI_CHAR || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_SHORT, &mpi_type))) + if ((ret = find_mpi_type(PIO_SHORT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_SHORT) + if (mpi_type != MPI_SHORT || type_size != 2) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_INT, &mpi_type))) + if ((ret = find_mpi_type(PIO_INT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_INT) + if (mpi_type != MPI_INT || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type))) + if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_FLOAT) + if (mpi_type != MPI_FLOAT || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type))) + if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_DOUBLE) + if (mpi_type != MPI_DOUBLE || type_size != 8) return ERR_WRONG; + /* These should also work. */ + if ((ret = find_mpi_type(PIO_INT, &mpi_type, NULL))) + return ret; + if (mpi_type != MPI_INT) + return ERR_WRONG; + if ((ret = find_mpi_type(PIO_INT, NULL, &type_size))) + return ret; + if (type_size != 4) + return ERR_WRONG; + if ((ret = find_mpi_type(PIO_INT, NULL, NULL))) + return ret; + #ifdef _NETCDF4 - if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type))) + if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_CHAR) + if (mpi_type != MPI_UNSIGNED_CHAR || type_size != 1) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_USHORT, &mpi_type))) + if ((ret = find_mpi_type(PIO_USHORT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_SHORT) + if (mpi_type != MPI_UNSIGNED_SHORT || type_size != 2) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_UINT, &mpi_type))) + if ((ret = find_mpi_type(PIO_UINT, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED) + if (mpi_type != MPI_UNSIGNED || type_size != 4) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_INT64, &mpi_type))) + if ((ret = find_mpi_type(PIO_INT64, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_LONG_LONG) + if (mpi_type != MPI_LONG_LONG || type_size != 8) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_UINT64, &mpi_type))) + if ((ret = find_mpi_type(PIO_UINT64, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_UNSIGNED_LONG_LONG) + if (mpi_type != MPI_UNSIGNED_LONG_LONG || type_size != 8) return ERR_WRONG; - if ((ret = find_mpi_type(PIO_STRING, &mpi_type))) + if ((ret = find_mpi_type(PIO_STRING, &mpi_type, &type_size))) return ret; - if (mpi_type != MPI_CHAR) + if (mpi_type != MPI_CHAR || type_size != 1) return ERR_WRONG; #endif /* _NETCDF4 */ @@ -461,6 +345,117 @@ int test_misc() /* This should not work. */ if (flush_buffer(TEST_VAL_42, &wmb, 0) != PIO_EBADID) return ERR_WRONG; + + return 0; +} + +/* This test code was recovered from main() in pioc_sc.c. */ +int test_CalcStartandCount() +{ + int ndims = 2; + int gdims[2] = {31, 777602}; + int num_io_procs = 24; + bool converged = false; + PIO_Offset start[ndims], kount[ndims]; + int iorank, numaiotasks = 0; + long int tpsize = 0; + long int psize; + long int pgdims = 1; + int scnt; + int ret; + + for (int i = 0; i < ndims; i++) + pgdims *= gdims[i]; + + while (!converged) + { + for (iorank = 0; iorank < num_io_procs; iorank++) + { + if ((ret = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank, + start, kount, &numaiotasks))) + return ret; + if (iorank < numaiotasks) + printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0], + start[1], kount[0], kount[1]); + + if (numaiotasks < 0) + return numaiotasks; + + psize = 1; + scnt = 0; + for (int i = 0; i < ndims; i++) + { + psize *= kount[i]; + scnt += kount[i]; + } + tpsize += psize; + } + + if (tpsize == pgdims) + converged = true; + else + { + printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs); + tpsize = 0; + num_io_procs--; + } + } + + return 0; +} + +/* Test the GDCblocksize() function. */ +int run_GDCblocksize_tests(MPI_Comm test_comm) +{ + { + int arrlen = 1; + PIO_Offset arr_in[1] = {0}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 2, 3}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 4) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 2, 3, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 3, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } + + { + int arrlen = 4; + PIO_Offset arr_in[4] = {0, 1, 2, 4}; + PIO_Offset blocksize; + + blocksize = GCDblocksize(arrlen, arr_in); + if (blocksize != 1) + return ERR_WRONG; + } return 0; } @@ -482,10 +477,20 @@ int main(int argc, char **argv) * nothing. */ if (my_rank < TARGET_NTASKS) { + /* I don't need this iosystem, but it's the only way to get + * the logs to write. */ + int iosysid; + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, 1, 0, PIO_REARR_BOX, &iosysid))) + return ret; + printf("%d running tests for functions in pioc_sc.c\n", my_rank); if ((ret = run_sc_tests(test_comm))) return ret; + printf("%d running tests for GCDblocksize()\n", my_rank); + if ((ret = run_GDCblocksize_tests(test_comm))) + return ret; + printf("%d running spmd test code\n", my_rank); if ((ret = run_spmd_tests(test_comm))) return ret; @@ -498,18 +503,6 @@ int main(int argc, char **argv) if ((ret = test_lists())) return ret; - printf("%d running rearranger opts tests 1\n", my_rank); - if ((ret = test_rearranger_opts1())) - return ret; - - printf("%d running rearranger opts tests 2\n", my_rank); - if ((ret = test_rearranger_opts2())) - return ret; - - printf("%d running compare_offsets tests\n", my_rank); - if ((ret = test_compare_offsets())) - return ret; - printf("%d running ceil2/pair tests\n", my_rank); if ((ret = test_ceil2_pair())) return ret; @@ -522,6 +515,10 @@ int main(int argc, char **argv) if ((ret = test_misc())) return ret; + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ diff --git a/src/externals/pio2/tests/general/ncdf_get_put.F90.in b/src/externals/pio2/tests/general/ncdf_get_put.F90.in index 75ccc75f3e9b..ea8e5933b710 100644 --- a/src/externals/pio2/tests/general/ncdf_get_put.F90.in +++ b/src/externals/pio2/tests/general/ncdf_get_put.F90.in @@ -118,6 +118,67 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1datt PIO_TF_AUTO_TEST_SUB_END test_put_get_1datt +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_0dvar + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(var_desc_t) :: pio_var, pio_cvar + PIO_TF_FC_DATA_TYPE, dimension(1) :: pval, gval + CHARACTER(len=1) :: pcval, gcval + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, ret + + pval = pio_tf_world_sz_ + pcval = "D" + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_val', PIO_TF_DATA_TYPE, pio_var) + PIO_TF_CHECK_ERR(ret, "Failed to define scalar var:" // trim(filename)) + + ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_cval', PIO_char, pio_cvar) + PIO_TF_CHECK_ERR(ret, "Failed to define scalar char var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_var, pval); + PIO_TF_CHECK_ERR(ret, "Failed to put scalar var:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_cvar, pcval); + PIO_TF_CHECK_ERR(ret, "Failed to put scalar char var:" // trim(filename)) + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_var, gval); + PIO_TF_CHECK_ERR(ret, "Failed to get scalar var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval, pval), "Got wrong value") + + ret = PIO_get_var(pio_file, pio_cvar, gcval); + PIO_TF_CHECK_ERR(ret, "Failed to get scalar char var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gcval, pcval), "Got wrong value") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_0dvar + PIO_TF_TEMPLATE PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar Implicit none @@ -184,6 +245,71 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar +! Write out a 1d var slice from a 2d var +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_slice + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(var_desc_t) :: pio_var, pio_cvar + integer :: pio_dim + integer, parameter :: MAX_ROW_DIM_LEN = 100 + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN) :: gval, exp_val + integer, parameter :: MAX_COL_DIM_LEN = 4 + ! Only COL_WRITE_DIM of MAX_COL_DIM_LEN columns in pval is written out + integer, parameter :: COL_WRITE_DIM = 2 + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN, MAX_COL_DIM_LEN) :: pval + integer, dimension(:) :: start(4), count(4) + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, ret + + pval = -1 + pval(:,COL_WRITE_DIM) = pio_tf_world_sz_ + exp_val = pio_tf_world_sz_ + start = 0 + count = 0 + start(1) = 1 + count(1) = MAX_ROW_DIM_LEN + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put_slice.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', MAX_ROW_DIM_LEN, pio_dim) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ret = PIO_put_var(pio_file, pio_var, start, count, pval(:,COL_WRITE_DIM)); + PIO_TF_CHECK_ERR(ret, "Failed to put var:" // trim(filename)) + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_var, gval); + PIO_TF_CHECK_ERR(ret, "Failed to get var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval, exp_val), "Got wrong value") + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_slice + PIO_TF_TEMPLATE PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts Implicit none @@ -267,3 +393,177 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_4parts +! Write out 2d/3d/4d vars, one time slice at a time +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_md2mdplus1_var + Implicit none + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + integer, parameter :: MAX_DIMS = 4 + integer, parameter :: MAX_ROWS = 10 + integer, parameter :: MAX_COLS = 10 + integer, parameter :: MAX_LEVS = 3 + integer, parameter :: MAX_TIMES = 3 + integer, dimension(MAX_DIMS) :: pio_dims + type(var_desc_t) :: pio_2dvar, pio_3dvar, pio_4dvar + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_TIMES) :: gval_2d, exp_val_2d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_TIMES) :: gval_3d, exp_val_3d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_LEVS,MAX_TIMES) ::& + gval_4d, exp_val_4d + ! Only one slice is written out at a time + ! pval_1d is a 1d slice of gval_2d ... + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS) :: pval_1d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS) :: pval_2d + PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS, MAX_LEVS) :: pval_3d + integer, dimension(:) :: start(MAX_DIMS), count(MAX_DIMS) + integer :: pval_start + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + integer :: i, k, l, m, n, tstep, ret + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_ncdf_get_put_md_slice.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i) + ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename)) + + ! Since file is just created no need to enter redef + ret = PIO_def_dim(pio_file, 'nrows', MAX_ROWS, pio_dims(1)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'ncols', MAX_COLS, pio_dims(2)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'nlevs', MAX_LEVS, pio_dims(3)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_dim(pio_file, 'timesteps', MAX_TIMES, pio_dims(4)) + PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename)) + + ret = PIO_def_var(pio_file, '2d_val', PIO_TF_DATA_TYPE,& + (/pio_dims(1),pio_dims(4)/), pio_2dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_def_var(pio_file, '3d_val', PIO_TF_DATA_TYPE,& + (/pio_dims(1),pio_dims(2),pio_dims(4)/), pio_3dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_def_var(pio_file, '4d_val', PIO_TF_DATA_TYPE,& + pio_dims, pio_4dvar) + PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename)) + + ret = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename)) + + ! Put vals are for each timestep & + ! expected vals are combined for all timesteps + do k=1,MAX_ROWS + pval_1d(k) = k + end do + do tstep=1,MAX_TIMES + pval_start = (tstep - 1) * MAX_ROWS + exp_val_2d(:,tstep) = pval_1d + pval_start + end do + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_2d(k,l) = (l - 1)*MAX_ROWS + k + end do + end do + do tstep=1,MAX_TIMES + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS) + exp_val_3d(:,:,tstep) = pval_2d + pval_start + end do + end do + end do + do m=1,MAX_LEVS + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_3d(k,l,m) = ((m-1)*(MAX_COLS*MAX_ROWS)+(l - 1)*MAX_ROWS + k) + end do + end do + end do + do tstep=1,MAX_TIMES + do m=1,MAX_LEVS + do l=1,MAX_COLS + do k=1,MAX_ROWS + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS) + exp_val_4d(:,:,:,tstep) = pval_3d + pval_start + end do + end do + end do + end do + ! Put 2d/3d/4d vals, one timestep at a time + do tstep=1,MAX_TIMES + start = 0 + count = 0 + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = tstep + count(2) = 1 + pval_start = (tstep - 1) * MAX_ROWS + ret = PIO_put_var(pio_file, pio_2dvar, start, count,& + pval_1d(:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 2d var:" // trim(filename)) + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = 1 + count(2) = MAX_COLS + start(3) = tstep + count(3) = 1 + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS) + ret = PIO_put_var(pio_file, pio_3dvar, start, count,& + pval_2d(:,:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 3d var:" // trim(filename)) + + start(1) = 1 + count(1) = MAX_ROWS + start(2) = 1 + count(2) = MAX_COLS + start(3) = 1 + count(3) = MAX_LEVS + start(4) = tstep + count(4) = 1 + pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS) + ret = PIO_put_var(pio_file, pio_4dvar, start, count,& + pval_3d(:,:,:)+pval_start) + PIO_TF_CHECK_ERR(ret, "Failed to put 4d var:" // trim(filename)) + end do + + call PIO_syncfile(pio_file) + + ret = PIO_get_var(pio_file, pio_2dvar, gval_2d) + PIO_TF_CHECK_ERR(ret, "Failed to get 2d var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval_2d, exp_val_2d), "Got wrong value (2d var)") + + ret = PIO_get_var(pio_file, pio_3dvar, gval_3d) + PIO_TF_CHECK_ERR(ret, "Failed to get 3d var:" // trim(filename)) + + PIO_TF_CHECK_VAL((gval_3d, exp_val_3d), "Got wrong value (3d var)") + + ret = PIO_get_var(pio_file, pio_4dvar, gval_4d) + PIO_TF_CHECK_ERR(ret, "Failed to get 4d var:" // trim(filename)) + + ! Special code to handle 4d vals is required since the framework + ! currently does not support comparing 4d arrays + do tstep=1,MAX_TIMES + PIO_TF_CHECK_VAL((gval_4d(:,:,:,tstep), exp_val_4d(:,:,:,tstep)), "Got wrong value (4d var)") + end do + + call PIO_closefile(pio_file) + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + +PIO_TF_AUTO_TEST_SUB_END test_put_get_md2mdplus1_var + diff --git a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in index e92b483c3c02..e75fdce61de4 100644 --- a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +++ b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in @@ -225,3 +225,196 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_4d_col_decomp deallocate(rbuf) deallocate(wbuf) PIO_TF_AUTO_TEST_SUB_END nc_write_read_4d_col_decomp + +! Using a 3d decomp for writing out a 3d and a 4d var +! Write with one decomp (to force rearrangement) and read with another (no +! rearrangement) +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN nc_reuse_3d_decomp + implicit none + integer, parameter :: NDIMS = 4 + integer, parameter :: NFRAMES = 3 + type(var_desc_t) :: pio_var3d, pio_var4d + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: wr_iodesc, rd_iodesc + integer, dimension(:), allocatable :: compdof + integer, dimension(NDIMS) :: start, count + PIO_TF_FC_DATA_TYPE, dimension(:,:,:,:), allocatable :: rbuf4d, wbuf4d, exp_val4d + PIO_TF_FC_DATA_TYPE, dimension(:,:,:), allocatable :: rbuf3d, wbuf3d, exp_val3d + integer, dimension(NDIMS-1) :: dims + integer, dimension(NDIMS) :: pio_dims + integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts + integer(kind=pio_offset_kind) :: f + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Set the decomposition for writing data - forcing rearrangement + call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.) + nrows = count(1) + ncols = count(2) + nhgts = count(3) + + ! Initialize the 4d var + allocate(wbuf4d(nrows, ncols, nhgts, NFRAMES)) + do f=1,NFRAMES + do k=1,nhgts + do j=1,ncols + do i=1,nrows + wbuf4d(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + wbuf4d(i,j,k,f) = wbuf4d(i,j,k,f) + (f - 1) * (dims(1) * dims(2) * dims(3)) + end do + end do + end do + end do + allocate(compdof(nrows * ncols * nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i + compdof(tmp_idx) = wbuf4d(i,j,k,1) + end do + end do + end do + ! Initialize the 3d var + allocate(wbuf3d(nrows, ncols, nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + wbuf3d(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + end do + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) + deallocate(compdof) + + ! Set the decomposition for reading data - different from the write decomp + call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.) + nrows = count(1) + ncols = count(2) + nhgts = count(3) + + allocate(rbuf4d(nrows, ncols, nhgts, NFRAMES)) + rbuf4d = 0 + ! Expected val for 4d var + allocate(exp_val4d(nrows, ncols, nhgts, NFRAMES)) + do f=1,NFRAMES + do k=1,nhgts + do j=1,ncols + do i=1,nrows + exp_val4d(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + exp_val4d(i,j,k,f) = exp_val4d(i,j,k,f)+(f - 1) * (dims(1) * dims(2) * dims(3)) + end do + end do + end do + end do + allocate(compdof(nrows * ncols * nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i + compdof(tmp_idx) = exp_val4d(i,j,k,1) + end do + end do + end do + + allocate(rbuf3d(nrows, ncols, nhgts)) + rbuf3d = 0 + ! Expected val for 3d var + allocate(exp_val3d(nrows, ncols, nhgts)) + do k=1,nhgts + do j=1,ncols + do i=1,nrows + exp_val3d(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +& + (start(2) - 1 + j - 1) * dims(1) + i + end do + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc) + deallocate(compdof) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(4)) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_3d_var', PIO_TF_DATA_TYPE, pio_dims(1:3), pio_var3d) + PIO_TF_CHECK_ERR(ierr, "Failed to define a 3d var : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_4d_var', PIO_TF_DATA_TYPE, pio_dims, pio_var4d) + PIO_TF_CHECK_ERR(ierr, "Failed to define a 4d var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + call PIO_write_darray(pio_file, pio_var3d, wr_iodesc, wbuf3d, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write 3d darray : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var4d, f) + ! Write the current frame + call PIO_write_darray(pio_file, pio_var4d, wr_iodesc, wbuf4d(:,:,:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write 4d darray : " // trim(filename)) + end do + call PIO_syncfile(pio_file) + + rbuf4d = 0 + rbuf3d = 0 + + call PIO_read_darray(pio_file, pio_var3d, rd_iodesc, rbuf3d, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read 3d darray : " // trim(filename)) + + do f=1,NFRAMES + call PIO_setframe(pio_file, pio_var4d, f) + call PIO_read_darray(pio_file, pio_var4d, rd_iodesc, rbuf4d(:,:,:,f), ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read 4d darray : " // trim(filename)) + end do + + do f=1,NFRAMES + PIO_TF_CHECK_VAL((rbuf4d(:,:,:,f), exp_val4d(:,:,:,f)), "Got wrong 4d val, frame=", f) + end do + PIO_TF_CHECK_VAL((rbuf3d, exp_val3d), "Got wrong 3dd val") + + call PIO_closefile(pio_file) + + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc) + call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) + + deallocate(exp_val3d) + deallocate(rbuf3d) + deallocate(wbuf3d) + + deallocate(exp_val4d) + deallocate(rbuf4d) + deallocate(wbuf4d) +PIO_TF_AUTO_TEST_SUB_END nc_reuse_3d_decomp diff --git a/src/externals/pio2/tests/general/util/pio_tutil.F90 b/src/externals/pio2/tests/general/util/pio_tutil.F90 index 43c0b634b4a1..e4a076f3a101 100644 --- a/src/externals/pio2/tests/general/util/pio_tutil.F90 +++ b/src/externals/pio2/tests/general/util/pio_tutil.F90 @@ -129,7 +129,7 @@ SUBROUTINE PIO_TF_Init_(rearr) - pio_tf_log_level_ = 0 + pio_tf_log_level_ = 3 pio_tf_num_aggregators_ = 0 pio_tf_num_io_tasks_ = 0 pio_tf_stride_ = 1 diff --git a/src/share/unit_test_stubs/pio/pio.F90.in b/src/share/unit_test_stubs/pio/pio.F90.in index 9aafa3550dc6..a855889396bd 100644 --- a/src/share/unit_test_stubs/pio/pio.F90.in +++ b/src/share/unit_test_stubs/pio/pio.F90.in @@ -87,8 +87,9 @@ module pio integer,parameter,public :: PIO_rearr_comm_fc_1d_io2comp = 2 integer,parameter,public :: PIO_rearr_comm_fc_2d_disable = 3 integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1 + integer, public, parameter :: PIO_NOERR=0 - + public :: PIO_set_rearr_opts public :: PIO_def_dim public :: PIO_enddef public :: PIO_FILE_IS_OPEN @@ -273,6 +274,18 @@ contains type (io_desc_t) :: iodesc end subroutine freedecomp_file + integer function PIO_set_rearr_opts(iosystem, comm_type, fcd,& + enable_hs_c2i, enable_isend_c2i,& + max_pend_req_c2i,& + enable_hs_i2c, enable_isend_i2c,& + max_pend_req_i2c) result(ierr) + type (iosystem_desc_t), intent(inout) :: iosystem + integer, intent(in) :: comm_type, fcd + logical, intent(in) :: enable_hs_c2i, enable_hs_i2c + logical, intent(in) :: enable_isend_c2i, enable_isend_i2c + integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c + end function PIO_set_rearr_opts + integer function get_att_desc_{TYPE} (File,varDesc,name,value) result(ierr) type (File_desc_t), intent(inout) , target :: File type (VAR_desc_t), intent(in) :: varDesc diff --git a/src/share/util/shr_pio_mod.F90 b/src/share/util/shr_pio_mod.F90 index 2e02ed83c523..27894e9e73b6 100644 --- a/src/share/util/shr_pio_mod.F90 +++ b/src/share/util/shr_pio_mod.F90 @@ -52,7 +52,11 @@ module shr_pio_mod integer, allocatable :: io_compid(:) integer :: pio_debug_level=0, pio_blocksize=0 integer(kind=pio_offset_kind) :: pio_buffer_size_limit=-1 - type(pio_rearr_opt_t) :: pio_rearr_opts + integer :: pio_rearr_opt_comm_type, pio_rearr_opt_fcd + logical :: pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend + integer :: pio_rearr_opt_c2i_max_pend_req + logical :: pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend + integer :: pio_rearr_opt_i2c_max_pend_req integer :: total_comps=0 #define DEBUGI 1 @@ -144,6 +148,7 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia integer :: ncomps character(len=shr_kind_cl) :: nlfilename, cname type(iosystem_desc_t) :: iosys + integer :: ret character(*), parameter :: subName = '(shr_pio_init2) ' if(pio_debug_level>0) then @@ -176,12 +181,18 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia allocate(iosystems(total_comps)) if(pio_async_interface) then -#ifdef PIO1 - call pio_init(total_comps,mpi_comm_world, comp_comm, io_comm, iosystems, rearr_opts=pio_rearr_opts) - -#else call pio_init(total_comps,mpi_comm_world, comp_comm, io_comm, iosystems) -#endif + do i=1,total_comps + ret = pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,& + pio_rearr_opt_fcd,& + pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,& + pio_rearr_opt_c2i_max_pend_req,& + pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,& + pio_rearr_opt_i2c_max_pend_req) + if(ret /= PIO_NOERR) then + write(shr_log_unit,*) "ERROR: Setting rearranger options failed" + end if + end do i=1 else do i=1,total_comps @@ -199,7 +210,16 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, & pio_comp_settings(i)%pio_stride, & pio_comp_settings(i)%pio_rearranger, iosystems(i), & - base=pio_comp_settings(i)%pio_root, rearr_opts=pio_rearr_opts) + base=pio_comp_settings(i)%pio_root) + ret = pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,& + pio_rearr_opt_fcd,& + pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,& + pio_rearr_opt_c2i_max_pend_req,& + pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,& + pio_rearr_opt_i2c_max_pend_req) + if(ret /= PIO_NOERR) then + write(shr_log_unit,*) "ERROR: Setting rearranger options failed" + end if if(comp_comm_iam(i)==0) then write(shr_log_unit,*) io_compname(i),' : pio_numiotasks = ',pio_comp_settings(i)%pio_numiotasks write(shr_log_unit,*) io_compname(i),' : pio_stride = ',pio_comp_settings(i)%pio_stride @@ -374,9 +394,9 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root, integer :: iam, ierr, npes, unitn logical :: iamroot - namelist /pio_default_inparm/ pio_stride, pio_root, pio_numiotasks, & - pio_typename, pio_async_interface, pio_debug_level, pio_blocksize, & - pio_buffer_size_limit, pio_rearranger, & + namelist /pio_default_inparm/ & + pio_async_interface, pio_debug_level, pio_blocksize, & + pio_buffer_size_limit, & pio_rearr_comm_type, pio_rearr_comm_fcd, & pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, & pio_rearr_comm_enable_isend_comp2io, & @@ -408,6 +428,15 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root, pio_async_interface = .false. ! pio tasks are a subset of component tasks pio_rearranger = PIO_REARR_SUBSET + pio_rearr_comm_type = 'p2p' + pio_rearr_comm_fcd = '2denable' + pio_rearr_comm_max_pend_req_comp2io = 0 + pio_rearr_comm_enable_hs_comp2io = .true. + pio_rearr_comm_enable_isend_comp2io = .false. + pio_rearr_comm_max_pend_req_io2comp = 0 + pio_rearr_comm_enable_hs_io2comp = .true. + pio_rearr_comm_enable_isend_io2comp = .false. + if(iamroot) then unitn=shr_file_getunit() open( unitn, file=trim(nlfilename), status='old' , iostat=ierr) @@ -782,29 +811,29 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd, ! buf(6) = max_pend_req_io2comp ! buf(7) = enable_hs_io2comp ! buf(8) = enable_isend_io2comp - pio_rearr_opts%comm_type = buf(1) - pio_rearr_opts%fcd = buf(2) - pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = buf(3) + pio_rearr_opt_comm_type = buf(1) + pio_rearr_opt_fcd = buf(2) + pio_rearr_opt_c2i_max_pend_req = buf(3) if(buf(4) == 0) then - pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .false. + pio_rearr_opt_c2i_enable_hs = .false. else - pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .true. + pio_rearr_opt_c2i_enable_hs = .true. end if if(buf(5) == 0) then - pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .false. + pio_rearr_opt_c2i_enable_isend = .false. else - pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .true. + pio_rearr_opt_c2i_enable_isend = .true. end if - pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req = buf(6) + pio_rearr_opt_i2c_max_pend_req = buf(6) if(buf(7) == 0) then - pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .false. + pio_rearr_opt_i2c_enable_hs = .false. else - pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .true. + pio_rearr_opt_i2c_enable_hs = .true. end if if(buf(8) == 0) then - pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .false. + pio_rearr_opt_i2c_enable_isend = .false. else - pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true. + pio_rearr_opt_i2c_enable_isend = .true. end if end subroutine !=============================================================================== diff --git a/src/share/util/shr_strdata_mod.F90 b/src/share/util/shr_strdata_mod.F90 index 01dda274f6df..323e08fb9429 100644 --- a/src/share/util/shr_strdata_mod.F90 +++ b/src/share/util/shr_strdata_mod.F90 @@ -1165,7 +1165,7 @@ subroutine shr_strdata_readnml(SDAT,file,rc,mpicom) call MPI_COMM_SIZE(mpicom,ntasks,rCode) endif -!--master--task-- + !--master--task-- if (my_task == master_task) then !---------------------------------------------------------------------------- @@ -1241,15 +1241,21 @@ subroutine shr_strdata_readnml(SDAT,file,rc,mpicom) end do do n = 1,SDAT%nstreams - call shr_stream_parseInput(SDAT%streams(n),fileName,yearAlign,yearFirst,yearLast) - call shr_stream_init(SDAT%stream(n),fileName,yearFirst,yearLast,yearAlign, & - trim(SDAT%taxMode(n))) + if (trim(SDAT%streams(n)) /= shr_strdata_nullstr) then + + ! extract fileName (stream description text file), yearAlign, yearFirst, yearLast from SDAT%streams(n) + call shr_stream_parseInput(SDAT%streams(n), fileName, yearAlign, yearFirst, yearLast) + + ! initialize stream datatype, read description text file + call shr_stream_init(SDAT%stream(n), fileName, yearFirst, yearLast, yearAlign, trim(SDAT%taxMode(n))) + + end if enddo -! call shr_strdata_print(SDAT,trim(file)//' NML_ONLY') + ! call shr_strdata_print(SDAT,trim(file)//' NML_ONLY') endif ! master_task -!--master--task-- + !--master--task-- if (present(mpicom)) then call shr_strdata_bcastnml(SDAT,mpicom) diff --git a/tools/configure b/tools/configure index bba14b82d700..997b1d3e524d 100755 --- a/tools/configure +++ b/tools/configure @@ -69,7 +69,7 @@ def parse_command_line(args): argcnt = len(args) args = parser.parse_args() - CIME.utils.handle_standard_logging_options(args) + CIME.utils.parse_args_and_handle_standard_logging_options(args) opts = {} if args.machines_dir is not None: diff --git a/tools/cprnc/README b/tools/cprnc/README index 1ac0fd2e0902..eceb705aafc5 100644 --- a/tools/cprnc/README +++ b/tools/cprnc/README @@ -19,14 +19,14 @@ configure. Next, run make to build cprnc. For instance, using sh/bash as a login shell: -CIMEROOT=../.. source .env_mach_specific.sh && make +CIMEROOT=../.. source ./.env_mach_specific.sh && make Finally, put the resulting executable in CCSM_CPRNC as defined in config_machines.xml. You can also build cprnc using cmake: export CIMEROOT=../.. -MPILIB=mpi-serial source .env_mach_specific.sh +MPILIB=mpi-serial source ./.env_mach_specific.sh ../configure --macros-format=CMake Next run cmake . to build the Makefile and then diff --git a/tools/cprnc/summarize_cprnc_diffs b/tools/cprnc/summarize_cprnc_diffs index 1d6b5cf8d224..8adaf87b75fd 100755 --- a/tools/cprnc/summarize_cprnc_diffs +++ b/tools/cprnc/summarize_cprnc_diffs @@ -105,7 +105,10 @@ SYNOPSIS it looks for files of the form *.nc.cprnc.out.SUFFIX. (With this naming convention [i.e., looking for files of the form *.nc.cprnc.out], note that it only looks at output for baseline comparisons - NOT output from the test - itself, such as cprnc output files from the exact restart test.) + itself, such as cprnc output files from the exact restart test.) (Actually, + we also allow for files of the form *.nc_[0-9][0-9][0-9][0-9].cprnc.out, + such as *.nc_0001.cprnc.out and *.nc_0002.cprnc.out, to pick up + multi-instance files.) Summaries of cprnc differences (RMS and normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs) are placed in three output files beginning with the name 'cprnc.summary', in @@ -148,10 +151,10 @@ sub process_cprnc_output { my @cprnc_files; if ($output_suffix) { - @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix}"; + @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix} ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out.${output_suffix}"; } else { - @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out"; + @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out"; } foreach my $cprnc_file (@cprnc_files) { diff --git a/tools/mapping/check_maps/.gitignore b/tools/mapping/check_maps/.gitignore index cdfc1c16f6e4..7d65634c86f7 100644 --- a/tools/mapping/check_maps/.gitignore +++ b/tools/mapping/check_maps/.gitignore @@ -1,2 +1,4 @@ ESMF_RegridWeightGenCheck src/*.o +*.nc +*.Log diff --git a/tools/mapping/check_maps/README b/tools/mapping/check_maps/README index f1731dddc390..0970d3fc28b4 100644 --- a/tools/mapping/check_maps/README +++ b/tools/mapping/check_maps/README @@ -1,9 +1,3 @@ -========================================================================== -$Id: README 46983 2013-05-09 22:08:12Z tcraig $ -$URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/check_maps/README $ -========================================================================== - - =============== ABOUT THIS TOOL =============== @@ -42,23 +36,29 @@ BUILD ===== To compile this tool, you need to have the ESMFMKFILE environment variable set. -The easiest way to do that is to load the ESMF modules. To do this, you must -already have the intel module loaded. Then load the version of ESMF you want -to use (as of April 2013, the most recent version available is 6.1.1): +The easiest way to do that is to load the ESMF modules. On yellowstone and +cheyenne, you must already have the intel module loaded. Then load the version +of ESMF you want to use (as of April 2017, the most recent version available is +7.0.0): +Yellowstone: $ module load esmf -$ module load esmf-6.3.0r-ncdfio-uni-O +$ module load esmf-7.0.0-ncdfio-uni-O + +Cheyenne: +$ module load esmf_libs/7.0.0 +$ module load esmf-7.0.0-ncdfio-uni-O -This will set ESMFMKFILE. With this variable set, enter the mapping/check_maps/src/ -directory and run gmake +This will set ESMFMKFILE. With this variable set, enter the src/ directory and +run gmake -$ cd mapping/check_maps/src +$ cd src $ gmake [VERBOSE=TRUE] VERBOSE=TRUE turns on some extra diagnostic output and is optional. -Back in the mapping/check_maps/ directory, you should now have an executable -file named ESMF_RegridWeightGenCheck. +Back in this directory, you should now have an executable file named +ESMF_RegridWeightGenCheck. Note that at this time, the tool only works in serial mode (building with the mpi-enabled version of the ESMF makefile leads to a segmentation fault when @@ -78,6 +78,6 @@ listed in FILELIST --help, -h Output this usage information Notes: - 1) For use on yellowstone, geyser, or caldera only! + 1) For use on cheyenne, yellowstone, geyser, or caldera only! 2) Need to set ESMFMKFILE (see comments in Makefile) or compilation will fail 3) If -rc option is not enabled, -v flag is ignored and verbose / concise will depend on previous compilation diff --git a/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 b/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 index 189cc70ba6fd..01c500ea1b5d 100644 --- a/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 +++ b/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 @@ -716,6 +716,8 @@ program OfflineTester failCnt = failCnt + 1 if (Verbose) then print *, "FAILED: conservation error = ", totArea + print *, " area1 = ", sum(grid1area) + print *, " area2 = ", sum(grid2area) else print *, "FAILED: conservation error = ", totArea, & " in test ", j diff --git a/tools/mapping/check_maps/src/Makefile b/tools/mapping/check_maps/src/Makefile index 873af453426b..2d1ddb3ba757 100644 --- a/tools/mapping/check_maps/src/Makefile +++ b/tools/mapping/check_maps/src/Makefile @@ -15,11 +15,18 @@ ### The executable should run on the yellowstone login node ###### ################################################################################ -ifneq ($(origin ESMFMKFILE), environment) -$(error Environment variable ESMFMKFILE was not set.) +# Don't require ESMF to be loaded to run "make clean": +USE_ESMF=TRUE +ifeq ($(MAKECMDGOALS),clean) + USE_ESMF=FALSE endif -include $(ESMFMKFILE) +ifeq ($(USE_ESMF),TRUE) + ifneq ($(origin ESMFMKFILE), environment) + $(error Environment variable ESMFMKFILE was not set.) + endif + include $(ESMFMKFILE) +endif TARGET = ESMF_RegridWeightGenCheck # To compile with verbose output, run diff --git a/tools/mapping/gen_domain_files/INSTALL b/tools/mapping/gen_domain_files/INSTALL index a7b97822f2ee..a244892d2969 100644 --- a/tools/mapping/gen_domain_files/INSTALL +++ b/tools/mapping/gen_domain_files/INSTALL @@ -2,17 +2,16 @@ HOW TO BUILD ============ -Prior to building, you must make sure environment variables CIMEROOT and CIME_MODEL are set. - (1) $ cd src -(2) $ $CIMEROOT/tools/configure --machine [machine name] --macros-format Makefile -(3) Bash users: - $ . .env_mach_specific.sh - csh users: - $ source .env_mach_specific.csh -(4) $ gmake +(2) $ ../../../configure --macros-format Makefile --mpilib mpi-serial +Bash users: +(3) $ (. ./.env_mach_specific.sh ; gmake) +csh users: +(3) $ (source ./.env_mach_specific.csh ; gmake) -Note: in the second step, replace [machine name] with the machine you are -building on. Also, some machines have dedicated build nodes, so you might need -to SSH to another node before the 'gmake' step. +Note: in the second step, you may need to include "--machine [machine name]", +where [machine name] is the name of the machine you are building on. In most +cases configure can figure that out on its own, but if you get an error that is +the first fix to try. Also, some machines have dedicated build nodes, so you +might need to SSH to another node before the 'gmake' step. diff --git a/tools/mapping/gen_domain_files/src/Makefile b/tools/mapping/gen_domain_files/src/Makefile index 0a10703350c9..82c749eb39a4 100644 --- a/tools/mapping/gen_domain_files/src/Makefile +++ b/tools/mapping/gen_domain_files/src/Makefile @@ -36,7 +36,19 @@ default: $(EXENAME) $(MACROS): @echo "use the configure script located in the Machines directory to create the Makefile $(MACROS) file" --include $(MACROS) +# Do not include Macros for clean or distclean targets +USE_MACROS=TRUE +ifeq ($(MAKECMDGOALS),clean) + USE_MACROS=FALSE +endif +ifeq ($(MAKECMDGOALS),distcleanclean) + USE_MACROS=FALSE +endif + +ifeq ($(USE_MACROS),TRUE) + -include $(MACROS) +endif + # Check for the netcdf library and include directories ifdef NETCDF_PATH LIB_NETCDF:=$(NETCDF_PATH)/lib @@ -133,7 +145,7 @@ $(EXENAME): $(OBJS) clean: $(RM) -f $(OBJS) $(EXENAME) *.mod -veryclean: - $(RM) -f $(OBJS) $(EXENAME) *.mod $(MACROS) env_mach* Depends* .env_mach* +distclean: clean + ../../../configure --clean gen_domain.o : gen_domain.F90 diff --git a/tools/mapping/gen_mapping_files/README b/tools/mapping/gen_mapping_files/README index cd60a919454d..a9594191d3a3 100644 --- a/tools/mapping/gen_mapping_files/README +++ b/tools/mapping/gen_mapping_files/README @@ -120,8 +120,8 @@ Note: if rtm is specified and lnd is not, then this tool will You can also set the following env variables: ESMFBIN_PATH - Path to ESMF binaries - (Leave unset on yellowstone and caldera and the tool - will be loaded from modules) + (Leave unset on cheyenne, yellowstone, and caldera and the + tool will be loaded from modules) MPIEXEC ------ Name of mpirun executable (default is mpirun.lsf on yellowstone and caldera; if you run interactively on yellowstone, mpi is not used) diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore new file mode 100644 index 000000000000..4e025b24ca32 --- /dev/null +++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore @@ -0,0 +1,2 @@ +*.Log +*.nc diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README index c0f157a05c01..e0741bdb2d5c 100644 --- a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README +++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README @@ -82,8 +82,8 @@ where You can also set the following env variables: ESMFBIN_PATH - Path to ESMF binaries - (Leave unset on yellowstone and caldera and the tool - will be loaded from modules) + (Leave unset on cheyenne, yellowstone, and caldera and the + tool will be loaded from modules) MPIEXEC ------ Name of mpirun executable (default is mpirun.lsf on yellowstone and caldera; if you run interactively on yellowstone, mpi is not used) diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh index 012fbe9e417d..60448bf1c8d1 100755 --- a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh +++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh @@ -80,8 +80,8 @@ usage() { echo '' echo 'You can also set the following env variables:' echo ' ESMFBIN_PATH - Path to ESMF binaries ' - echo ' (Leave unset on yellowstone/caldera/pronghorn and the tool' - echo ' will be loaded from modules)' + echo ' (Leave unset on cheyenne/yellowstone/caldera/pronghorn' + echo ' and the tool will be loaded from modules)' echo ' MPIEXEC ------ Name of mpirun executable' echo ' (default is mpirun.lsf on yellowstone/caldera/pronghorn; if' echo ' you run interactively on yellowstone, mpi is not used)' @@ -225,6 +225,9 @@ if [ $MACH == "UNSET" ]; then ys* ) MACH="yellowstone" ;; + cheyenne* ) + MACH="cheyenne" + ;; geyser* ) MACH="geyser" ;; @@ -244,10 +247,13 @@ if [ $MACH == "UNSET" ]; then fi # Machine specific settings: -# 1) can not run in parallel interactively on yellowstone +# 1) can not run in parallel interactively on yellowstone or cheyenne if [ $MACH == "yellowstone" ] && [ $interactive == "YES" ]; then serial="TRUE" fi +if [ $MACH == "cheyenne" ] && [ $interactive == "YES" ]; then + serial="TRUE" +fi # 2) jaguar requires additional environment var if [ $MACH == "jaguar" ] && [ -z "$REGRID_PROC" ]; then REGRID_PROC=8 @@ -310,22 +316,26 @@ fi case $MACH in ## yellowstone, geyser, caldera, or pronghorn - "yellowstone" | "geyser" | "caldera" | "pronghorn" ) + "cheyenne" | "yellowstone" | "geyser" | "caldera" | "pronghorn" ) # From tcsh, script will not find module command # So check to see if module works, otherwise source an init file module list > /dev/null 2>&1 || source /etc/profile.d/modules.sh module purge module load intel module load nco - module load esmf + if [ $MACH == "cheyenne" ]; then + module load esmf_libs/7.0.0 + else + module load esmf + fi if [ $serial == "TRUE" ]; then - module load esmf-6.3.0rp1-ncdfio-uni-O + module load esmf-7.0.0-ncdfio-uni-O if [ -z "$MPIEXEC" ]; then MPIEXEC="" fi else - module load esmf-6.3.0rp1-ncdfio-mpi-O + module load esmf-7.0.0-ncdfio-mpi-O if [ -z "$MPIEXEC" ]; then MPIEXEC="mpirun.lsf" fi diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL index d57d7b49545b..c4af56772286 100644 --- a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL +++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL @@ -5,9 +5,9 @@ HOW TO BUILD (1) $ cd src (2) $ ../../../../configure --macros-format Makefile --mpilib mpi-serial Bash users: -(3) $ (. .env_mach_specific.sh ; gmake) +(3) $ (. ./.env_mach_specific.sh ; gmake) csh users: -(3) $ (source .env_mach_specific.csh ; gmake) +(3) $ (source ./.env_mach_specific.csh ; gmake) Note: in the second step, you may need to include "--machine [machine name]", where [machine name] is the name of the machine you are building on. In most diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 index a415e633ee41..e04319015797 100644 --- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 +++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 @@ -208,6 +208,7 @@ PROGRAM main !--- create new map datatype to hold result of matrix-matrix multiply --- call map_dup(map_orig,map_new) map_new%title = trim(title) + map_new%domain_b = trim(map_smooth%domain_b) call map_matMatMult(map_orig,map_new,map_smooth) ! mult(A,B,S): B=S*A call mapsort_sort(map_new) call map_check(map_new) diff --git a/tools/mapping/map_field/INSTALL b/tools/mapping/map_field/INSTALL index 766c76e3fec0..a244892d2969 100644 --- a/tools/mapping/map_field/INSTALL +++ b/tools/mapping/map_field/INSTALL @@ -2,17 +2,16 @@ HOW TO BUILD ============ -Prior to building, you must make sure $CIMEROOT is set. - (1) $ cd src -(2) $ $CIMEROOT/tools/configure macros-format Makefile -(3) Bash users: - $ . .env_mach_specific.sh - csh users: - $ source .env_mach_specific.csh -(4) $ gmake +(2) $ ../../../configure --macros-format Makefile --mpilib mpi-serial +Bash users: +(3) $ (. ./.env_mach_specific.sh ; gmake) +csh users: +(3) $ (source ./.env_mach_specific.csh ; gmake) -Note: in the second step, replace [machine name] with the machine you are -building on. Also, some machines have dedicated build nodes, so you might need -to SSH to another node before the 'gmake' step. +Note: in the second step, you may need to include "--machine [machine name]", +where [machine name] is the name of the machine you are building on. In most +cases configure can figure that out on its own, but if you get an error that is +the first fix to try. Also, some machines have dedicated build nodes, so you +might need to SSH to another node before the 'gmake' step. diff --git a/tools/mapping/map_field/src/Makefile b/tools/mapping/map_field/src/Makefile index bc2dbd56e8ea..27b53b6128fa 100644 --- a/tools/mapping/map_field/src/Makefile +++ b/tools/mapping/map_field/src/Makefile @@ -29,14 +29,26 @@ null := CURDIR = . EXENAME = ../map_field RM = rm -MACFILE = Macros.make +MACROS = Macros.make default: $(EXENAME) -$(MACFILE): - @echo "use the configure script located in the Machines directory to create the Makefile $(MACFILE) file" +$(MACROS): + @echo "use the configure script located in the Machines directory to create the Makefile $(MACROS) file" + +# Do not include Macros for clean or distclean targets +USE_MACROS=TRUE +ifeq ($(MAKECMDGOALS),clean) + USE_MACROS=FALSE +endif +ifeq ($(MAKECMDGOALS),distcleanclean) + USE_MACROS=FALSE +endif + +ifeq ($(USE_MACROS),TRUE) + -include $(MACROS) +endif --include $(MACFILE) # Check for the netcdf library and include directories ifdef NETCDF_PATH LIB_NETCDF:=$(NETCDF_PATH)/lib @@ -97,7 +109,7 @@ else FPPDEFS := $(CPPDEFS) endif #Primary Target: build the tool -all: $(EXENAME) $(MACFILE) +all: $(EXENAME) $(MACROS) OBJS := map_field.o @@ -133,7 +145,7 @@ $(EXENAME): $(OBJS) clean: $(RM) -f $(OBJS) $(EXENAME) *.mod -veryclean: - $(RM) -f $(OBJS) $(EXENAME) *.mod $(MACFILE) .env_mach* env_mach* +distclean: clean + ../../../configure --clean map_field.o : map_field.F90 diff --git a/tools/unit_testing/python/comparable.py b/tools/unit_testing/python/comparable.py deleted file mode 100644 index 20e52697dba7..000000000000 --- a/tools/unit_testing/python/comparable.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Portable implementation of comparisons for total ordering. - -THIS MODULE IS OBSOLETE AS OF PYTHON 2.7. - -This module provides a "Comparable" class for subclassing; it provides the -various "rich comparison" functions (e.g. __le__) based on the subclass's -__eq__ and __lt__ methods (so this is a Template Method pattern). - -Python 2.7 and Python 3 implement a class decorator for this in the -standard libraries (functools.total_ordering), and that decorator is much -more flexible (besides being standard), so this should only be used for -compatibility with older implementations. -""" - -class Comparable(object): - - """Provides rich comparisons for subclasses that have __eq__ and __lt__.""" - - def __le__(self, other): - return (self == other) and (self < other) - - def __gt__(self, other): - return not (self <= other) - - def __ge__(self, other): - return not (self < other) - - def __ne__(self, other): - return not (self == other) diff --git a/tools/unit_testing/python/environment.py b/tools/unit_testing/python/environment.py deleted file mode 100644 index 62972c6ad574..000000000000 --- a/tools/unit_testing/python/environment.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Produce commands for environment module systems. - -Public classes: -EnvSystemInterface - Abstract base class for interfaces. -NoModuleInterface - Can be constructed, but raises exception if used. -ModuleInterface - For standard module systems (including Lmod). -SoftEnvInterface - For SoftEnv, the ANL MCS environment system. - -Public routines: -expand_env - Expand shell variables (like "${FOO}"). -""" - -import os -import re -import subprocess - -__all__ = ("EnvSystemInterface", "NoModuleInterface", "ModuleInterface", - "SoftEnvInterface", "expand_env") - -class EnvSystemInterface(object): - - """Abstract base class for environment system interfaces. - - This class exists only to document the interface of subclasses, and to - provide error messages if an unimplemented function is called by - accident. - - Normally, none of these methods should actually be called. All of them - raise NotImplementedError. - - Public methods: - is_loaded - purge - load - unload - purge_str - load_str - unload_str - list - """ - - @classmethod - def _raise_not_implemented(cls, method_name): - raise NotImplementedError( - cls.__name__ +" does not implement method "+method_name+".") - - def is_loaded(self, modname): - """Method not implemented.""" - self._raise_not_implemented("is_loaded") - - def purge(self): - """Method not implemented.""" - self._raise_not_implemented("purge") - - def list(self): - """Method not implemented.""" - self._raise_not_implemented("list") - - def load(self, modname): - """Method not implemented.""" - self._raise_not_implemented("load") - - def unload(self, modname): - """Method not implemented.""" - self._raise_not_implemented("unload") - - def purge_str(self): - """Method not implemented.""" - self._raise_not_implemented("purge_str") - - def load_str(self, modname): - """Method not implemented.""" - self._raise_not_implemented("load_str") - - def unload_str(self, modname): - """Method not implemented.""" - self._raise_not_implemented("unload_str") - - -class NoModuleInterface(EnvSystemInterface): - - """Module interface for systems with no module system. - - The purpose of this class is to allow code to construct a module - interface regardless of system, while still raising an error if the - user attempts to actually interact with the non-existent interface. - - As a result, only purge_str is implemented. - - Public methods: - purge_str - Returns null command. - """ - - def purge_str(self): - """Returns ":", the null shell command.""" - return ":" - - -class ModuleInterface(EnvSystemInterface): - - """Module interface for systems with a typical module system. - - Class methods: - python_init - Initialize python interface. - - Public methods: - is_loaded - Tests if a module is currently loaded. - purge - Purge all modules from environment. - load - Load a module. - unload - Unload a module. - list - list currently loaded modules. - purge_str - Return command to purge modules. - load_str - Return command to load the given module. - unload_str - Return command to unload the given module. - """ - - # Singleton declaring whether or not we have the module interface - # started up yet. - _python_initialized = False - - # Magic string to grep for in "module list" output to determine whether - # or not a module is present. - _not_loaded_string = "None found" - - @classmethod - def python_init(cls, filename): - """Initialize python to module system interface. - - This must be called before any commands that are intended to - modify python's environment. The *_str methods can still be called - without this. Only the first call has any effect. - - Arguments: - filename - Python file to execute to load the interface. - """ - if not cls._python_initialized: - execfile(filename, globals()) - cls._python_initialized = True - - def is_loaded(self, modname): - """"Whether the module is loaded in the current environment.""" - # This assertion message isn't quite true, but this behavior is - # convenient for testing purposes, which is why is_loaded exists. - assert self._python_initialized, \ - "Can't test modules without initializing the python interface!" - process = subprocess.Popen("module list "+modname, - shell=True, - env=os.environ, - stderr=subprocess.PIPE) - stderr_output = process.communicate()[1] - # Handle unexpected error in subprocess. - if process.returncode not in (0,1): - raise Exception("module list command failed with code: "+ - str(process.returncode)) - # Hack! Assume that process.returncode == 1 means that no modules - # are loaded, rather than some other possible error. - return process.returncode != 1 and \ - re.search(self._not_loaded_string, stderr_output) is None - - def purge(self): - """"Purge all modules from the current environment.""" - assert self._python_initialized, \ - "Can't purge modules without initializing the python interface!" - module("purge") - - def list(self): - """"List all modules from the current environment.""" - assert self._python_initialized, \ - "Can't list modules without initializing the python interface!" - module("list") - - def load(self, modname): - """"Load a module in the current environment.""" - assert self._python_initialized, \ - "Can't load modules without initializing the python interface!" - module("load", modname) - - def unload(self, modname): - """"Unload a module from the current environment.""" - assert self._python_initialized, \ - "Can't unload modules without initializing the python interface!" - module("unload", modname) - - def purge_str(self): - """Returns the shell command to purge modules as a string.""" - return "module purge" - - def load_str(self, modname): - """Returns the shell command to load the module as a string.""" - return "module load "+modname - - def unload_str(self, modname): - """Returns the shell command to unload the module as a string.""" - return "module unload "+modname - - -class SoftEnvInterface(EnvSystemInterface): - - """Module interface for systems with SoftEnv. - - Public methods: - purge_str - Return command to reset SoftEnv. - load_str - Return command to add the given keyword/macro. - unload_str - Return command to delete the given keyword/macro. - """ - - def purge_str(self): - """Returns the shell command to reset SoftEnv as a string. - - Unfortunately, there's not a straightforward way to clear - everything, so right now we just run "resoft". If it becomes - necessary in the future, this may change to do something else (e.g. - resoft using a custom file with only default settings). - """ - return "resoft" - - def load_str(self, modname): - """Returns the shell command to add the keyword as a string.""" - return "soft add "+modname - - def unload_str(self, modname): - """Returns the shell command to delete the keyword as a string.""" - return "soft delete "+modname - - -# Regex that matches an environment variable reference, putting the name in -# the "name" group of the match. -_env_re = re.compile( - """\$ # Initial "$" - (?P\{)? # Open brace if present - (?P[A-Za-z0-9_]+) # Variable name - (?(brace)\}) # Close brace if necessary""", re.X) - - -def expand_env(string, env): - """Expand a shell string with given environment. - - Arguments: - string - String to expand as if in a shell. - env - dict specifying environment variables. - - This is very limited; right now it only handles variable substitution, - and only with "${FOO}" or "$FOO" style syntax. The variable name must - contain only alphanumeric characters and "_". There is no way to escape - a "$". - - Expansion is recursive; the output of this function is its fixed point. - """ - def expand_func(match): - """Given an environment variable match, return replacement text.""" - var_name = match.group("name") - if var_name in env: - return env[var_name] - else: - # If there's no match in the environment, return original text. - return match.group(0) - old_string = "" - new_string = string - while new_string != old_string: - old_string = new_string - new_string = _env_re.sub(expand_func, new_string) - - return new_string diff --git a/tools/unit_testing/python/machine_setup.py b/tools/unit_testing/python/machine_setup.py deleted file mode 100644 index 1059beca258c..000000000000 --- a/tools/unit_testing/python/machine_setup.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Functions to set up machine-specific settings. - -Currently, this module is not under unit test, because it interacts too -directly with the OS of the machine it is on. Any changes to address this -would be welcome (this would likely require the use of mock modules, -and/or a specific set of machines to test on). - -Public classes: -MachineCompilerSettings - Set up machine/compiler specific environment. -""" - -# Python 3 compatible printing in Python 2. -from __future__ import print_function - -from itertools import chain -from os import environ -import os.path -import platform -import re -import subprocess -from xml.etree.ElementTree import ElementTree - -import environment as ENV -from printer import ScriptPrinter -from xml_utils import best_match, all_matches -__all__ = ("MachineCompilerSettings") - - -def get_machine_name(): - """Returns a canonical version of the machine name.""" - name = platform.node() - # Strip everything after the first ".", and whitespace. - name = name.split(".")[0].strip() - if re.match("^yslogin[0-9]+", name): - name = "yellowstone" - elif re.match("^miralac[0-9]+", name): - name = "mira" - elif re.match("^cetuslac[0-9]+", name): - name = "mira" - elif re.match("^caldera.*", name) or re.match("^pronghorn.*", name): - # Use yellowstone settings for caldera/pronghorn, since the mahcines - # files aren't set up explicitly for those machines, and they have the - # same configuration as yellowstone. - name = "yellowstone" - return name - -def load_machine_env(compiler): - """Add machine environment variables not in config_compilers.xml - - Besides simply setting variables, this may also load some modules. - """ - - mach = get_machine_name() - - if mach == "yellowstone": - mod = ENV.ModuleInterface() - mod.python_init("/glade/apps/opt/lmod/lmod/init/env_modules_python.py") - mod.purge() - mod.load("ncarenv/1.0") - mod.load("ncarbinlibs/1.0") - if compiler == "intel": - mod.load("intel/15.0.1") - mod.load("mkl/11.1.2") - elif compiler == "pgi": - mod.load("pgi/13.9") - mod.load("ncarcompilers/1.0") - mod.load("cmake/2.8.10.2") - mod.load("netcdf/4.3.0") - - -class MachineCompilerSettings(object): - - """Encapsulate machine settings and set environment from them. - - Public methods: - __init__ - Discover information about local machine. - compiler_xml_to_env - Set a specific variable using config_compilers. - set_compiler_env - Set up machine-specific environment. - write_cmake_macros - Create CMake "Macros" file. - """ - - def __init__(self, compiler, compiler_xml_path, - machine=None, - use_env_compiler=False, - mpilib=None, use_openmp=False): - """Discover information about the machine and compiler. - - Arguments: - compiler - String naming the compiler vendor. - compiler_xml_path - Path to config_compilers.xml file. - machine - String naming the machine (guessed if not provided) - use_env_compiler - Force use of environment variable FC instead of - using the compiler in config_compilers.xml. - mpilib - String naming the mpi library if any. - use_openmp - Boolean option to use OpenMP compiler flags. - Currently only works for CESM/CESM_DEBUG builds. - """ - if machine is None: - machine = get_machine_name() - self.machine_dict = { - "COMPILER": compiler, - "MACH": machine, - "OS": platform.system(), - "compile_threaded": "true" if use_openmp else "false", - } - if mpilib: - self.machine_dict["MPILIB"] = mpilib - else: - self.machine_dict["MPILIB"] = 'mpi-serial' - - self.compiler_xml_tree = ElementTree() - self.compiler_xml_tree.parse(compiler_xml_path) - self.use_env_compiler = use_env_compiler - self.mpilib = mpilib - - def compiler_xml_to_env(self, xml_path, var_name): - """Look up a config_compilers entry and set a variable from it. - - Arguments: - xml_path - Path within the xml file (e.g. "compiler/SFC"). - var_name - Name of environment variable (e.g. "FC"). - """ - match = best_match(self.compiler_xml_tree, xml_path, - self.machine_dict) - assert match is not None, "Could not determine "+var_name+ \ - " from compiler/machines xml file." - environ[var_name] = match.text.strip() - - def add_path(self, name, macros_printer): - match = best_match(self.compiler_xml_tree, "compiler/"+name+"_PATH", - self.machine_dict) - if match is not None: - macros_printer.print_header(name + " location.") - libpath = match.text - _make_env_re = re.compile( - """\$\( # Initial "$" and brace - (?P[A-Za-z0-9_]+) # Variable name - \) # Close brace""", re.X) - libpath = _make_env_re.sub( "$ENV{\g}",libpath) - print("libpath = "+libpath) - macros_printer.print( - "list(APPEND CMAKE_PREFIX_PATH "+libpath+")" - ) - - def set_compiler_env(self): - """Set up the environment for this machine and compiler.""" - if not self.use_env_compiler: - if (self.mpilib == "mpi-serial" or self.mpilib is None): - self.compiler_xml_to_env("compiler/SFC", "FC") - self.compiler_xml_to_env("compiler/SCC", "CC") - else: - self.compiler_xml_to_env("compiler/MPIFC", "FC") - self.compiler_xml_to_env("compiler/MPICC", "CC") - load_machine_env(self.machine_dict["COMPILER"]) - - def write_cmake_macros(self, macros_file, model="CESM"): - """Write CMake macros file using config_compilers.xml - - Arguments: - macros_file - File object to write to. - """ - - # Print header to file. - macros_printer = ScriptPrinter(macros_file) - header_lines = [model+ - " build flags for:", - " Compiler = "+self.machine_dict["COMPILER"], - " Machine = "+self.machine_dict["MACH"], - " OS = "+self.machine_dict["OS"], - " MPILIB = "+self.machine_dict["MPILIB"], - ] - - for line in header_lines: - macros_printer.comment(line) - - macros_printer.print("include(Compilers)") - macros_printer.print( - "set(CMAKE_Fortran_FLAGS_"+model+" \"\" CACHE STRING \"Flags used by the Fortran compiler during builds.\" FORCE)") - macros_printer.print( - "set(CMAKE_Fortran_FLAGS_"+model+"_DEBUG \"\" CACHE STRING \"Flags used by the Fortran compiler during DEBUG builds.\" FORCE)") - - - macros_printer.print( - "set(CMAKE_C_FLAGS_"+model+" \"\" CACHE STRING \"Flags used by the C compiler during builds.\" FORCE)") - macros_printer.print( - "set(CMAKE_C_FLAGS_"+model+"_DEBUG \"\" CACHE STRING \"Flags used by the C compiler during DEBUG builds.\" FORCE)") - - macros_printer.print( - "mark_as_advanced(CMAKE_Fortran_FLAGS_"+model+" CMAKE_Fortran_FLAGS_" - +model+"_DEBUG)") - macros_printer.print( - "mark_as_advanced(CMAKE_C_FLAGS_"+model+" CMAKE_C_FLAGS_"+model+"_DEBUG)") - macros_printer.print( - "set(all_build_types \"None Debug Release RelWithDebInfo MinSizeRel "+model+ - " "+model+"_DEBUG\")") - macros_printer.print( - "set(CMAKE_BUILD_TYPE \"${CMAKE_BUILD_TYPE}\" CACHE STRING \"Choose the type of build, options are: ${all_build_types}.\" FORCE)") - - - - - # pFUnit location if it exists. - self.add_path("PFUNIT",macros_printer) - # NETCDF location if it exists. - self.add_path("NETCDF",macros_printer) - # PNETCDF location if it exists. - self.add_path("PNETCDF", macros_printer) - # HDF5 location if it exists. - self.add_path("HDF5", macros_printer) - # MPI location if it exists. - self.add_path("MPI", macros_printer) - # PETSc location if it exists. - self.add_path("PETSC", macros_printer) - # TRILINOS location if it exists. - self.add_path("TRILINOS", macros_printer) - # ALBANY location if it exists. - self.add_path("ALBANY", macros_printer) - - - # Normal and debug dictionaries for looking things up in - # config_compilers. - normal_dict = self.machine_dict.copy() - normal_dict["DEBUG"] = "FALSE" - - debug_dict = self.machine_dict.copy() - debug_dict["DEBUG"] = "TRUE" - - def add_formatted_flags(flags_name, format): - """Print CMake flags using macros_printer. - - Arguments: - flags_name - Name to search for in config_compilers. - format - Function that takes a build type and flag match, and - returns the string to print out. - """ - - paths = ["compiler/"+flags_name, "compiler/ADD_"+flags_name] - - # This creates an iterable over elements in config_compilers - # that match in non-debug mode. - normal_matches = chain.from_iterable( - all_matches(self.compiler_xml_tree, path, normal_dict) - for path in paths - ) - for match in normal_matches: - macros_printer.print(format(model, match.text)) - - # Now the same for debug mode. - debug_matches = chain.from_iterable( - all_matches(self.compiler_xml_tree, path, debug_dict) - for path in paths - ) - for match in debug_matches: - macros_printer.print(format(model+"_DEBUG", match.text)) - - # Below, we just use a bunch of lambda functions to describe how - # the build type and a matching element (e.g. an FFLAGS entry) are - # turned into a CMake function call. - - macros_printer.print_header("CPP definitions.") - add_formatted_flags( - "CPPDEFS", - lambda b, m: "add_config_definitions("+b+" "+m+")" - ) - def format_contiguous(build_type, match): - comma = "," if self.machine_dict["COMPILER"] != "ibm" else "\\\," - contig_def = "contiguous"+comma if match == "TRUE" else "" - return "add_config_definitions("+build_type+\ - " -DUSE_CONTIGUOUS="+contig_def+")" - add_formatted_flags( - "HAS_F2008_CONTIGUOUS", - format_contiguous - ) - - macros_printer.print_header("Fortran flags.") - add_formatted_flags( - "FFLAGS", - lambda b, m: "add_flags(CMAKE_Fortran_FLAGS_"+b+" "+m+")" - ) - - macros_printer.print_header("C flags.") - add_formatted_flags( - "CFLAGS", - lambda b, m: "add_flags(CMAKE_C_FLAGS_"+b+" "+m+")" - ) - - macros_printer.print_header("Linker flags.") - add_formatted_flags( - "LDFLAGS", - lambda b, m: "add_flags(CMAKE_EXE_LINKER_FLAGS_"+b+" "+m+")" - ) - macros_printer.print_header("External library flags.") - add_formatted_flags( - "SLIBS", - lambda b, m: "add_flags(CMAKE_EXE_LINKER_FLAGS_"+b+" "+m+")" - ) diff --git a/tools/unit_testing/python/test_environment.py b/tools/unit_testing/python/test_environment.py deleted file mode 100755 index a7cbaa731024..000000000000 --- a/tools/unit_testing/python/test_environment.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python -"""Unit tests for the environment module. - -Public classes: -TestEnvSys - Test the EnvSystemInterface abstract base class. -TestNoMod - NoModuleInterface tests. -TestMod - ModuleInterface tests. -TestSoft - SoftEnvInterface tests. -TestExpandEnv - expand_env tests. - -To test the actual module system, you must specify these environment -variables when running the test: -MODULE_SYSTEM - Type of module system available locally. -TEST_MODULE - Name of a module which is *not* loaded, to use for testing - purposes. -MODULE_FILE - File that must be executed to load the interface to python - (if any). -""" - -from os import environ -import unittest - -from environment import * - -__all__ = ("TestEnvSys", "TestNoMod", "TestMod", "TestSoft", - "TestExpandEnv") - -# Test data gleaned from the environment. -module_system = "none" -test_module = "foo" -module_file = None -if "MODULE_SYSTEM" in environ: - module_system = environ["MODULE_SYSTEM"] - assert "TEST_MODULE" in environ, \ - "MODULE_SYSTEM is set, but TEST_MODULE is not defined." - test_module = environ["TEST_MODULE"] - if module_system == "module": - assert "MODULE_FILE" in environ, \ - "MODULE_SYSTEM is module, but MODULE_FILE is not defined." - module_file = environ["MODULE_FILE"] - - -class TestEnvSys(unittest.TestCase): - - """Tests for the EnvSystemInterface class. - - This tests that each method is NotImplemented. Other test classes in - this module inherit from this one in order to detect when a method has - been implemented but no test is defined for the new method. - - However, figuring out which class causes the error is a bit of a - guessing game. Addressing that issue is difficult in Python 2.6 because - of how assertRaises works. - """ - - test_class = EnvSystemInterface - - def setUp(self): - """Set up by creating an instance of the class under test.""" - self.test_obj = self.test_class() - - def test_is_loaded(self): - """is_loaded raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.is_loaded, test_module) - - def test_purge(self): - """purge raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.purge) - - def test_load(self): - """load raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.load, test_module) - - def test_purge_str(self): - """purge_str raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.purge_str) - - def test_load_str(self): - """load_str raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.load_str, test_module) - - def test_unload_str(self): - """unload_str raises NotImplementedError.""" - self.assertRaises(NotImplementedError, - self.test_obj.unload_str, test_module) - - -class TestNoMod(TestEnvSys): - - """Tests for the NoModuleInterface class.""" - - test_class = NoModuleInterface - - def test_purge_str(self): - """User can call NoModuleInterface.purge_str and get a ":".""" - self.assertEqual(self.test_obj.purge_str(), ":") - - -class TestMod(TestEnvSys): - - """Tests for the ModuleInterface class. - - It's not easy to verify that this behaves correctly, so in many cases - we do some kind of check for self consistency only. If module_system - is not "module", then most of the tests look for an exception. - """ - - test_class = ModuleInterface - - def setUp(self): - """Set up module system if present, then call parent setUp.""" - if module_system == "module": - ModuleInterface.python_init(module_file) - super(TestMod, self).setUp() - - def tearDown(self): - """Remove the test module from the current environment.""" - if module_system == "module": - self.test_obj.unload(test_module) - - def test_is_loaded(self): - """ModuleInterface.is_loaded returns false for unloaded module.""" - if module_system == "module": - self.assertFalse(self.test_obj.is_loaded(test_module)) - else: - self.assertRaises(AssertionError, - self.test_obj.is_loaded, test_module) - - def test_purge(self): - """ModuleInterface.purge removes the test module.""" - if module_system == "module": - self.test_obj.load(test_module) - self.test_obj.purge() - self.assertFalse(self.test_obj.is_loaded(test_module)) - else: - self.assertRaises(AssertionError, - self.test_obj.purge) - - def test_load(self): - """ModuleInterface.load loads a module.""" - if module_system == "module": - self.test_obj.load(test_module) - self.assertTrue(self.test_obj.is_loaded(test_module)) - else: - self.assertRaises(AssertionError, - self.test_obj.load, test_module) - - def test_unload(self): - """ModuleInterface.unload unloads a module.""" - if module_system == "module": - self.test_obj.load(test_module) - self.test_obj.unload(test_module) - self.assertFalse(self.test_obj.is_loaded(test_module)) - else: - self.assertRaises(AssertionError, - self.test_obj.unload, test_module) - - def test_purge_str(self): - """User gets non-null string from ModuleInterface.purge_str.""" - self.assertNotEqual(self.test_obj.purge_str().strip(), - "") - - def test_load_str(self): - """User gets non-null string from ModuleInterface.load_str.""" - self.assertNotEqual(self.test_obj.load_str(test_module).strip(), - "") - - def test_unload_str(self): - """User gets non-null string from ModuleInterface.unload_str.""" - self.assertNotEqual(self.test_obj.unload_str(test_module).strip(), - "") - - -class TestSoft(TestEnvSys): - - """Tests for the SoftEnvInterface class. - - It's not easy to verify that this behaves correctly without a specific - test machine in mind. As with the module tests, we really just verify - that no exceptions are raised and the return isn't the null string. - """ - - test_class = SoftEnvInterface - - def test_purge_str(self): - """User gets non-null string from SoftEnvInterface.purge_str.""" - self.assertNotEqual(self.test_obj.purge_str().strip(), "") - - def test_load_str(self): - """User gets non-null string from SoftEnvInterface.load_str.""" - self.assertNotEqual(self.test_obj.load_str(test_module).strip(), - "") - - def test_unload_str(self): - """User gets non-null string from SoftEnvInterface.unload_str.""" - self.assertNotEqual(self.test_obj.unload_str(test_module).strip(), - "") - - -class TestExpandEnv(unittest.TestCase): - - """Tests for the expand_env function.""" - - def test_no_variable(self): - """With no variables, expand_env is a no-op.""" - self.assertEqual(expand_env("foo", {"UNUSED": "not used"}), "foo") - - def test_variable_missing(self): - """With variables not present, expand_env is a no-op.""" - self.assertEqual(expand_env("${NOT_HERE}", {}), "${NOT_HERE}") - - def test_brace_expansion(self): - """Test that an expansion works with curly braces.""" - self.assertEqual(expand_env("${FOO}", {"FOO": "bar"}), "bar") - - def test_bare_expansion(self): - """Test that an expansion works with no braces.""" - self.assertEqual(expand_env("$FOO", {"FOO": "bar"}), "bar") - - def test_recursive_expansion(self): - """Test that expansion is done recursively.""" - self.assertEqual(expand_env("${FOO}", - {"FOO": "${FOO2}", "FOO2": "bar"}), - "bar") - - def test_brace_closing(self): - """Test that braces must be closed for expansion to occur.""" - self.assertEqual(expand_env("${FOO", {"FOO": "bar"}), "${FOO") - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/unit_testing/python/test_xml_utils.py b/tools/unit_testing/python/test_xml_utils.py deleted file mode 100755 index f05a6bf72f1a..000000000000 --- a/tools/unit_testing/python/test_xml_utils.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python -"""Unit tests for the xml_utils module. - -Public classes: -TestBestMatch - best_match tests. -TestAllMatches - all_matches tests. -TestElementsToDict - elements_to_dict tests. -""" - -import unittest -from xml.etree.ElementTree import XML, ElementTree - -from xml_utils import best_match, all_matches, elements_to_dict - -__all__ = ("TestBestMatch", "TestAllMatches", "TestElementsToDict") - -class TestBestMatch(unittest.TestCase): - - """Tests for the best_match function.""" - - def setUp(self): - """Create an ElementTree object as test data. - - This is a complex set of data, but the tests on it are - comparatively simple. Some of the data (such as the empty "foo" - tags) are present purely to prove that they are never matched. - """ - - root = XML(""" - - - a - a - - - generic - generic - generic - - - b - b - - - ab - - - - - - - abar - - - bbar - - -""") - - self.xml_tree = ElementTree(root) - - def test_no_match(self): - """best_match returns None when no paths match.""" - - self.assertTrue(best_match(self.xml_tree, "invalid") - is None) - - def test_simple_match(self): - """best_match can find the only match, when no attributes are used.""" - - self.assertEqual( - "generic", - best_match(self.xml_tree, "foo/data1").text - ) - - def test_no_match_attr(self): - """best_match returns None when there are no attribute matches.""" - - self.assertTrue(best_match(self.xml_tree, "bar/data1") - is None) - self.assertTrue(best_match(self.xml_tree, "bar/data1", {"a": "1"}) - is None) - - def test_match_with_attr(self): - """best_match returns the only path match, with matching attribute.""" - - self.assertEqual( - "abar", - best_match(self.xml_tree, "bar/data1", {"a": "2"}).text - ) - - def test_match_on_attr(self): - """best_match returns the only path and attribute match.""" - - self.assertEqual( - "b", - best_match(self.xml_tree, "foo/data2", - {"a": "2", "b": "2"}).text - ) - - def test_match_most_specific(self): - """best_match returns the most specific match for each path.""" - - self.assertEqual( - "generic", - best_match(self.xml_tree, "foo/data1", - {"a": "1", "b": "2"}).text - ) - self.assertEqual( - "ab", - best_match(self.xml_tree, "foo/data3", - {"a": "1", "b": "2"}).text - ) - - def test_match_first(self): - """best_match returns the first matching entry.""" - - self.assertEqual( - "abar", - best_match(self.xml_tree, "bar/data1", - {"a": "2", "b": "1"}).text - ) - - -class TestAllMatches(unittest.TestCase): - - """Tests for the all_matches function.""" - - def setUp(self): - """Create an ElementTree for use as test data.""" - - root = XML(""" - - - the_text - - - first_text - second_text - - - - - first_text - second_text - second_text - - - first_text - second_text - third_text - fourth_text - bad_text - - - the_text - - -""") - - self.xml_tree = ElementTree(root) - - def test_one_match(self): - """all_matches returns one element successfully.""" - self.assertEqual( - [e.text for e in all_matches(self.xml_tree, "test1/data")], - ["the_text"], - ) - - def test_multiple_matches(self): - """all_matches returns many elements successfully. - - Elements must be returned in the right order as well. - """ - self.assertEqual( - [e.text for e in all_matches(self.xml_tree, "test2/data")], - ["first_text", "second_text"], - ) - - def test_no_matches(self): - """all_matches returns no elements if none are found.""" - self.assertEqual( - [e.text for e in all_matches(self.xml_tree, "test3/data")], - [], - ) - - def test_no_attr_matches(self): - """all_matches returns no elements if none match attributes.""" - self.assertEqual( - [e.text for e in all_matches(self.xml_tree, "test4/data", - {"valid": "true"})], - [], - ) - - def test_attr_matches(self): - """all_matches returns elements that match attributes.""" - self.assertEqual( - [e.text for e in - all_matches(self.xml_tree, "test5/data", - {"valid": "true", "invalid": "false"})], - ["first_text", "second_text", "third_text", "fourth_text"], - ) - - def test_ignore_attribute(self): - """all_matches can ignore "extra" attributes.""" - self.assertEqual( - [e.text for e in - all_matches(self.xml_tree, "test6/data", - ignore=["ignore"])], - ["the_text"] - ) - - -# In true TDD form, these tests are far longer than the code they apply to! -# Of course, when they were written, the first version of the code was -# longer. -class TestElementsToDict(unittest.TestCase): - - """Tests for the elements_to_dict function.""" - - @staticmethod - def string_to_elements(xml_str, ignore="key"): - """Converts an XML string to a list of its "ENV" elements.""" - return all_matches(ElementTree(XML(xml_str)), "ENV", - ignore=ignore) - - def test_no_elements(self): - """elements_to_dict with no elements produces an empty dict.""" - self.assertEqual(elements_to_dict([]), {}) - - def test_one_element(self): - """elements_to_dict can handle one element.""" - elements = self.string_to_elements(""" - - bar - -""") - self.assertEqual(elements_to_dict(elements), {"foo": "bar"}) - - def test_multiple_elements(self): - """elements_to_dict can handle multiple elements.""" - elements = self.string_to_elements(""" - - bar1 - bar2 - -""") - self.assertEqual(elements_to_dict(elements), - {"foo1": "bar1", "foo2": "bar2"}) - - def test_key_attr(self): - """elements_to_dict uses key_attr as key attribute.""" - key_attr = "name" - xml_str = """ - - bar - -""" - elements = self.string_to_elements(xml_str, ignore=key_attr) - self.assertEqual(elements_to_dict(elements, key_attr=key_attr), - {"foo": "bar"}) - - def test_multiple_key(self): - """elements_to_dict lets later values overwrite earlier ones. - - This might not actually be the best behavior, but it's the easiest - to implement; this test is really just to prevent *accidentally* - changing the design. - """ - elements = self.string_to_elements(""" - - bar1 - bar2 - -""") - self.assertEqual(elements_to_dict(elements), - {"foo": "bar2"}) - - def test_keyless_match(self): - """elements_to_dict with a match with no key returns empty dict. - - It might be better to raise an exception; this is a robustness and - simplicity vs. correctness tradeoff. - """ - elements = self.string_to_elements(""" - - bar - -""") - self.assertEqual(elements_to_dict(elements), {}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/unit_testing/python/xml_utils.py b/tools/unit_testing/python/xml_utils.py deleted file mode 100644 index 4089c669afee..000000000000 --- a/tools/unit_testing/python/xml_utils.py +++ /dev/null @@ -1,202 +0,0 @@ -"""XML search utilities. - -These are based on the standard library xml.etree.ElementTree module. -However, that module is not explicitly imported at this time, so it is in -principle possible to pass arguments that imitate the interface of that -module. - -Exported functions: -best_match - Search for a specific element in an XML tree. -all_matches - Search for all matching elements in an XML tree. -elements_to_dict - Create a dict from XML entries with a key attribute. -""" - -from comparable import Comparable - -__all__ = ("best_match", "all_matches", "elements_to_dict") - -class ElementMatch(Comparable): - - """Class to aid searching for/matching an xml element. - - Public methods: - __init__ - Create an ElementMatch. - __iadd__ - Add in "quality" of other match. - __eq__ - Test if quality is same as another match. - __lt__ - Compare quality to another match. - __nonzero__/__bool__ - Test for a valid match. - - Public data: - element - Element for the match (None if no match). Read-only. - - Note that using __eq__ to test quality interferes with hashable - collections, so for now, __hash__ = None. - """ - - def __init__(self, element=None, quality=0): - """Define a element match. - - Arguments: - element - The element that was found. - quality - Abstract measure of "quality" (specificity) of the match. - - With both options left out, this will return a null ElementMatch. - """ - self._element = element - self._quality = quality - - def __iadd__(self, other): - """Add the quality from another match to that of this match.""" - self._quality += other._quality - return self - - __hash__ = None - """Not implemented, because __eq__ is overridden.""" - - def __eq__(self, other): - """Check whether two matches have the same quality.""" - return (not self and not other) or \ - (self and other and self._quality == other._quality) - - def __lt__(self, other): - """Compare matches by their quality; null matches always lose.""" - return (not self and other) or \ - (self and other and self._quality < other._quality) - - def __bool__(self): - """Test if this is a real match or just the null one.""" - return self._element is not None - - __nonzero__ = __bool__ - - @property - def element(self): - """Element from a valid match, or None for a null match.""" - return self._element - -# Right now this is treated as private to the module. But it's getting big -# enough to justify having its own tests, maybe. -def _element_attribute_match(element, attributes, ignore=[]): - """Check an element to see if it matches the given attributes. - - If an element passes the check, give it a "quality" corresponding to - the number of matched attributes. Otherwise, return null match. - - The ignore attribute specifies attributes to ignore. - """ - match_quality = 0 - for key in element.keys(): - if key in ignore: - continue - if key in attributes and attributes[key] == element.get(key): - match_quality += 1 - else: - return ElementMatch() - return ElementMatch(element, match_quality) - -def best_match(xml_tree, path, attributes={}): - """Find the best match for a path with attributes in an XML tree. - - The return value is the matched element. - - Arguments: - xml_tree - A tree from the xml.etree.ElementTree module. - path - The path to search for. - attributes - A dict containing attributes to match. Not all attributes - must be present on an element to get a match, but any - attributes present must match this input, and the "best" - match has the most matches. - - The attributes argument defaults to an empty dict. - """ - - # Recursive function to find a match. - def find_best_below(element, path): - # Done when there's no more of the path to match. - if len(path) == 0: - return ElementMatch(element) - - # Otherwise, get the beginning part of the path. - path_head, slash, path_tail = path.partition("/") - - # Search through subelements that match the next part of the path. - # For each one, get the quality of the match based on the number - # of matching attributes, then call self recursively to get - # best match from subelements. - best_match = ElementMatch() - for trial_element in element.findall(path_head): - local_match = _element_attribute_match(trial_element, - attributes) - if local_match: - new_match = \ - find_best_below(trial_element, path_tail) - new_match += local_match - if best_match < new_match: - best_match = new_match - - return best_match - - # Do the search. - element = xml_tree.getroot() - match = find_best_below(element, path) - - return match.element - -def all_matches(xml_tree, path, attributes={}, ignore=[]): - """Find all matches for a path with attributes in an XML tree. - - This is a generator. Each of the returned values will be a matching - element. - - Arguments: - xml_tree - A tree from the xml.etree.ElementTree module. - path - The path to search for. - attributes - A dict containing attributes to match. Not all attributes - must be present on an element to get a match, but any - attributes present must match this input. - ignore - A list of attributes to ignore when matching. Attributes in - this list are ignored regardless of whether or not they are in - the attributes dict. - - The attributes argument defaults to an empty dict. - """ - - # Recursive function to find all matches. - def find_matches_below(element, path): - # Done when there's no more of the path to match. - if len(path) == 0: - yield ElementMatch(element) - return - - # Otherwise, get the beginning part of the path. - path_head, slash, path_tail = path.partition("/") - - # Search through subelements that match the next part of the path. - # For each one, call self recursively to get all subelement matches. - for trial_element in element.findall(path_head): - if _element_attribute_match(trial_element, attributes, ignore): - for match in find_matches_below(trial_element, path_tail): - yield match - - element = xml_tree.getroot() - for match in find_matches_below(element, path): - yield match.element - -def elements_to_dict(elements, key_attr="key"): - """Uses a key attribute to produce a dict from ElementTree elements. - - For each element, it creates an entry in the returned dict. The key is - determined by key_attr, and the value is the text of the element. - - Arguments: - elements - An iterable of elements to convert. - key_attr - The name of an attribute. Each element must have this - attribute to be included in the dict, and the value of this - attribute will be used as the key. - """ - - return dict( - (elem.get(key_attr), elem.text) - for elem in elements - if key_attr in elem.keys() - ) diff --git a/utils/perl5lib/Build/Namelist.pm b/utils/perl5lib/Build/Namelist.pm index 5c222c1fab18..cd3643067d26 100644 --- a/utils/perl5lib/Build/Namelist.pm +++ b/utils/perl5lib/Build/Namelist.pm @@ -627,9 +627,10 @@ my $valint = "[+-]?[0-9]+"; my $valint_repeat = "${valint}\\*$valint"; # Logical data. -my $vallogical1 = "\\.[Tt][Rr][Uu][Ee]\\."; -my $vallogical2 = "\\.[Ff][Aa][Ll][Ss][Ee]\\."; -my $vallogical = "$vallogical1|$vallogical2"; +my $vallogical1 = "\\.?[Tt][Rr][Uu][Ee]\\.?"; +my $vallogical2 = "\\.?[Ff][Aa][Ll][Ss][Ee]\\.?"; +my $vallogical3 = "[FfTt]"; +my $vallogical = "$vallogical1|$vallogical2|$vallogical3"; my $vallogical_repeat = "${valint}\\*$vallogical1|${valint}\\*$vallogical2"; # Real data.