diff --git a/.codacy.yml b/.codacy.yml index bc0c61d106..8043fd6d0b 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,3 +1,4 @@ exclude_paths: - 'tests/extern/**' - 'lib/**/*' + - 'docs/**/*' diff --git a/README.md b/README.md index 9fc5065bef..11ec62284d 100644 --- a/README.md +++ b/README.md @@ -29,126 +29,15 @@ they can be automatically migrated and thereby executed on varying hardware resources without explicit programmer mapping, location, and communication management. -## Building - -*vt* can be built with `cmake` or built inside a `docker` container. - -To build *vt*, one must obtain the following dependencies: - -### Optional: - -#### If threading is enabled: - - OpenMP _or_ - - `std::thread`s (default to from C++ compiler) - -#### Required: - - detector, (*vt* ecosystem) - - checkpoint, (*vt* ecosystem) - - MPI (mpich/openmpi/mvapich/IBM Spectrum MPI/Cray MPICH/etc.) - -### Automatically build dependencies - -Assuming MPI is installed and accessible via CC/CXX, the only other dependencies -that are required are checkpoint and detector. The easiest way to get these -built are to clone them inside `vt/lib`: - -```bash -$ git clone git@github.com:DARMA-tasking/vt -$ cd vt/lib -$ git clone git@github.com:DARMA-tasking/checkpoint -$ git clone git@github.com:DARMA-tasking/detector -``` - -With these in `vt/lib`, cmake will automatically build them and stitch them into -*vt*'s linking process. - -Instead of running `cmake`, one may invoke the `vt/ci/build_cpp.sh` script which -will run `cmake` for *vt* with environment variables for most configuration -parameters. - -#### Environment Variables - -| Variable | Default Value | Description | -| ------------------ | --------------- | ----------- | -| `CMAKE_BUILD_TYPE` | Release | The `cmake` build type | -| `VT_LB_ENABLED` | ON | Enable compile-time load balancing support | -| `VT_TRACE_ENABLED ` | OFF | Enable compile-time tracing support | -| `VT_TRACE_RUNTIME_ENABLED ` | OFF | Force tracing on at runtime (used in CI for automatically testing tracing on all tests/examples) | -| `VT_DOXYGEN_ENABLED ` | OFF | Enable doxygen generation | -| `VT_MIMALLOC_ENABLED ` | OFF | Enable `mimalloc`, alternative allocator for debugging memory usage/frees/corruption | -| `VT_ASAN_ENABLED ` | OFF | Enable building with address sanitizer | -| `VT_POOL_ENABLED ` | ON | Use memory pool in *vt* for message allocation | -| `VT_ZOLTAN_ENABLED ` | OFF | Build with Zoltan enabled for `ZoltanLB` support | -| `ZOLTAN_DIR ` | (empty) | Directory pointing to Zoltan installation | -| `VT_MPI_GUARD_ENABLED ` | OFF | Enable compile-time load balancing support | - -With these set, invoke the script with two arguments: the path to the *vt* root directory and the build path. Here's an example assuming that *vt* is cloned into `/usr/src/vt` with trace enabled in debug mode. - -**Usage for building:** +## Read the documentation -```bash -$ vt/ci/build_cpp.sh -``` +To learn *vt*, read the [full +documentation](https://darma-tasking.github.io/docs/html/index.html) that is +automatically generated whenever a push occurs to "develop". It includes a +walk-though of the tutorial and a overview of the components that make up a *vt* +runtime. -**Example:** - -```bash -$ cd /usr/src -$ git clone git@github.com:DARMA-tasking/vt -$ VT_TRACE_ENABLED=1 CMAKE_BUILD_TYPE=Debug /usr/src/vt/ci/build_cpp.sh /usr/src/vt /usr/build/vt -``` - -### Building with `docker` containerization - -The easiest way to build *vt* is by using `docker` with the available containers that contain the proper compilers, MPI, and all other dependencies. First, install `docker` on the system. On some systems, `docker-compose` might also need to be installed. - -The `docker` builds are configured through `docker-compose` to use a shared, cached filesystem mount with the host for `ccache` to enable fast re-builds. - -For `docker-compose`, the following variables can be set to configure the build. One may configure the architecture, compiler type and version, Linux distro (ubuntu or alpine), and distro version. - -``` -# Variables: -# ARCH={amd64, arm64v8, ...} -# COMPILER_TYPE={gnu, clang} -# COMPILER={gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10, -# clang-3.9, clang-4.0, clang-5.0, clang-6.0, clang-7, clang-8, -# clang-9, clang-10} -# REPO=lifflander1/vt -# UBUNTU={18.04, 20.04} -# ULIMIT_CORE=0 -# -# DARMA/vt Configuration Variables: -# VT_LB=1 # Enable load balancing -# VT_TRACE=0 # Enable tracing -# VT_MIMALLOC=0 # Enable mimalloc memory allocator -# VT_DOCS=0 # Enable doxygen build -# VT_TRACE_RT=0 # Enable tracing at runtime (for testing) -# VT_ASAN=0 # Enable address sanitizer -# BUILD_TYPE=release # CMake build type -``` - -With these set, run the following for a non-interactive build: - -```bash -$ cd vt -$ docker-compose run -e BUILD_TYPE=debug -e VT_TRACE=1 ubuntu-cpp -``` - -For an interactive build, where one can build, debug, and run `valgrind`, etc: - -```bash -$ cd vt -$ docker-compose run -e BUILD_TYPE=debug -e VT_TRACE=1 ubuntu-cpp-interactive -$ /vt/ci/build_cpp.sh /vt /build -$ /vt/ci/test_cpp.sh /vt /build -``` - -For more detailed information on configuring the docker build, read the documentation in `vt/docker-compose.yml`. - -## Testing - -After *vt* is built succesfully, one may invoke the tests several ways. One may run `make test` or `ninja test` (depending on the generator used) or `ctest`, to run all the tests. Alternatively, the tests can be run automatically from the CI script: +## Building -```bash -$ vt/ci/test_cpp.sh -``` \ No newline at end of file +[Learn how to build](https://darma-tasking.github.io/docs/html/vt-build.html) +*vt* with `cmake` or `docker`. diff --git a/cmake/load_doxygen.cmake b/cmake/load_doxygen.cmake index 6b5e74c3a0..df8921c338 100644 --- a/cmake/load_doxygen.cmake +++ b/cmake/load_doxygen.cmake @@ -9,8 +9,16 @@ if (${vt_doxygen_enabled}) set(doxygen_in ${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in) set(doxygen_out ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) + set(DOXYGEN_PROJECT_NAME "vt") + set(VERSION_MAJOR "1") + set(VERSION_MINOR "0") + set(VERSION_PATCH "0") set(DOXYGEN_INPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/") + set(DOXYGEN_DOCS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/docs/") + set(DOXYGEN_EXAMPLE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/examples/") + set(DOXYGEN_TUTORIAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tutorial/") set(DOXYGEN_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/docs/") + set(DOXYGEN_MAIN_PAGE "${CMAKE_CURRENT_SOURCE_DIR}/src/vt.md") set(DOXYGEN_INDEX_FILE ${DOXYGEN_OUTPUT_DIR}/xml/index.xml) configure_file(${doxygen_in} ${doxygen_out} @ONLY) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index de938cd61c..34ddeb8f38 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "@CMAKE_PROJECT_NAME@" +PROJECT_NAME = "@DOXYGEN_PROJECT_NAME@" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version @@ -44,7 +44,7 @@ PROJECT_NUMBER = @VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@ # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = +PROJECT_BRIEF = "(Virtual Transport)" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 @@ -464,7 +464,7 @@ EXTRACT_ALL = NO # be included in the documentation. # The default value is: NO. -EXTRACT_PRIVATE = NO +EXTRACT_PRIVATE = YES # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. @@ -814,7 +814,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = "@DOXYGEN_INPUT_DIR@" +INPUT = "@DOXYGEN_INPUT_DIR@" \ + "@DOXYGEN_DOCS_DIR@" # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -931,21 +932,22 @@ EXCLUDE_SYMBOLS = # that contain example code fragments that are included (see the \include # command). -EXAMPLE_PATH = +EXAMPLE_PATH = "@DOXYGEN_EXAMPLE_DIR@" \ + "@DOXYGEN_TUTORIAL_DIR@" # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. -EXAMPLE_PATTERNS = * +EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. -EXAMPLE_RECURSIVE = NO +EXAMPLE_RECURSIVE = YES # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the @@ -1007,7 +1009,7 @@ FILTER_SOURCE_PATTERNS = # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. -USE_MDFILE_AS_MAINPAGE = @doxy_main_page@ +USE_MDFILE_AS_MAINPAGE = "@DOXYGEN_MAIN_PAGE@" #--------------------------------------------------------------------------- # Configuration options related to source browsing diff --git a/docs/Doxyfile.in-mcss b/docs/Doxyfile.in-mcss index 8f44493429..4ade5ba9ca 100644 --- a/docs/Doxyfile.in-mcss +++ b/docs/Doxyfile.in-mcss @@ -1,6 +1,31 @@ @INCLUDE = Doxyfile GENERATE_HTML = YES GENERATE_XML = YES -XML_PROGRAMLISTING = NO +XML_PROGRAMLISTING = YES M_SHOW_UNDOCUMENTED = YES -EXTRACT_ALL = YES \ No newline at end of file +EXTRACT_ALL = YES + +M_LINKS_NAVBAR1 = \ + "introduction context active-messenger collection collective group location objgroup pipe pool rdmahandle registry scheduler term trace" \ + "learn vt-build tutorial examples" \ + "pages" \ + "namespaces namespacevt namespacevt_1_1collective namespacevt_1_1debug namespacevt_1_1group namespacevt_1_1location namespacevt_1_1messaging namespacevt_1_1objgroup namespacevt_1_1pipe namespacevt_1_1rdma namespacevt_1_1runtime namespacevt_1_1term namespacevt_1_1vrt" + +M_LINKS_NAVBAR2 = \ + "annotated" \ + "files" \ + "GitHub Checkpoint Detector LBAF Checkpoint Analyzer Documentation" + + +ALIASES += \ + "vt=vt" \ + "m_div{1}=@xmlonly@endxmlonly" \ + "m_enddiv=@xmlonly@endxmlonly" \ + "m_span{1}=@xmlonly@endxmlonly" \ + "m_endspan=@xmlonly@endxmlonly" \ + "m_class{1}=@xmlonly@endxmlonly" \ + "m_footernavigation=@xmlonly@endxmlonly" \ + "m_examplenavigation{2}=@xmlonly@endxmlonly" \ + "m_keywords{1}=@xmlonly@endxmlonly" \ + "m_keyword{3}=@xmlonly@endxmlonly" \ + "m_enum_values_as_keywords=@xmlonly@endxmlonly" diff --git a/docs/md/active-messenger.md b/docs/md/active-messenger.md new file mode 100644 index 0000000000..de246ae0ab --- /dev/null +++ b/docs/md/active-messenger.md @@ -0,0 +1,95 @@ + +\page active-messenger Active Messenger +\brief Asynchronous send/receive of messages + +The active messenger `vt::messaging::ActiveMessenger`, accessed via +`vt::theMsg()`, asynchronously sends and receives messages across nodes using +MPI internally. When sending a message, it uses the \vt registry to consistently +dispatch messages and data to handlers (function pointers, functors, or methods) +across nodes. + +Each message contains an envelope `vt::Envelope` to store meta-data associated +with the message, such as the destination and handler to trigger when it +arrives. Sending a message entails setting up the envelope, optionally +serializing the message (depending on whether the serialize overload is +present), and then using `MPI_Isend` to asynchronously transfer the bytes to the +destination node. On the receive side, the active messenger is always probing +for an incoming message and begins a transfer when it discovers one. The \vt +\subpage{scheduler} polls the active messenger to make progress on any incoming +messages. + +\section am-simple-example Sending a message + +\code{.cpp} +#include + +#include + +// Declare a serializable message +struct MyMsg : vt::Message { + using MessageParentType = vt::Message; + vt_msg_serialize_required(); // for vector + + MyMsg() = default; // default constructor for de-serialize + MyMsg(int in_val, std::vector const& in_vec) + : val(in_val), + my_vec(in_vec) + { } + + template + void serialize(SerializerT& s) { + MessageParentType::serialize(s); + s | val; + s | my_vec; + } + + int val = 0; + std::vector my_vec; +}; + +// Active function pointer +void myHandler(MyMsg* m) { + vt::NodeType this_node = vt::theContext()->getNode(); + fmt::print("{}: val={}, vec size={}\n", this_node, m->val, m->my_vec.size()); +} + +// Active functor +struct MyFunctor { + void operator()(MyMsg* m) { + vt::NodeType this_node = vt::theContext()->getNode(); + fmt::print("{}: val={}, vec size={}\n", this_node, m->val, m->my_vec.size()); + } +}; + +int main(int argc, char** argv) { + vt::initialize(argc, argv); + + vt::NodeType this_node = vt::theContext()->getNode(); + + if (this_node == 0) { + // spins in scheduler until termination of the enclosed work + vt::runInEpochRooted([=]{ + std::vector vec_to_send; + vec_to_send.push_back(29.); + vec_to_send.push_back(54.); + + auto msg = vt::makeMessage(10, vec_to_send); + vt::theMsg()->sendMsg(1, msg.get()); // send to node 1 + + auto msg2 = vt::makeMessage(11, vec_to_send); + vt::theMsg()->sendMsg(1, msg2.get()); // send to node 1 + }); + } + + vt::finalize(); + return 0; +} + +\endcode + +Program output: + +\code{.shell-session} +1: val=10, vec size=2 +1: val=11, vec size=2 +\endcode diff --git a/docs/md/building.md b/docs/md/building.md new file mode 100644 index 0000000000..cc4afba24a --- /dev/null +++ b/docs/md/building.md @@ -0,0 +1,180 @@ +\page vt-build How to Build +\brief Building DARMA/vt with cmake + +\vt can be built with `cmake` or built inside a `docker` container. Most of the +external dependencies come bundled with \vt for ease of compiling. + +\section how-to-build Building + +To build \vt, one must obtain the following dependencies: + +\subsection required-deps Required + - detector, (*vt* ecosystem) + - checkpoint, (*vt* ecosystem) + - MPI (mpich/openmpi/mvapich/IBM Spectrum MPI/Cray MPICH/etc.) + +\subsection optional-deps Optional (if threading enabled) + + - OpenMP _or_ + - Default to `std::thread` + +\subsection automatic-build-deps Automatically build dependencies + +Assuming MPI is installed and accessible via CC/CXX, the only other dependencies +that are required are checkpoint and detector. The easiest way to get these +built are to clone them inside `vt/lib`: + +```bash +$ git clone git@github.com:DARMA-tasking/vt +$ cd vt/lib +$ git clone git@github.com:DARMA-tasking/checkpoint +$ git clone git@github.com:DARMA-tasking/detector +``` + +With these in `vt/lib`, cmake will automatically build them and stitch them into +*vt*'s linking process. + +\subsection use-cmake-directly-vars Using cmake directly + +One may use `cmake` as normal on *vt*, with checkpoint and detector cloned in +`vt/lib` to compile them all together as explained above. The following are some +custom configuration build options that can be provided to `cmake` to change the +build configuration: + +| CMake Variable | Default Value | Description | +| ------------------ | --------------- | ----------- | +| `vt_lb_enabled` | 0 | Compile with support for runtime load balancing | +| `vt_trace_enabled` | 0 | Compile with support for runtime tracing (Projections-format) | +| `vt_test_trace_runtime_enabled` | 0 | Force tracing on at runtime for VT tests | +| `vt_doxygen_enabled` | 0 | Enable doxygen generation | +| `vt_mimalloc_enabled` | 0 | Enable `mimalloc`, alternative allocator for debugging memory usage/frees/corruption | +| `vt_asan_enabled` | 0 | Enable building with address sanitizer | +| `vt_pool_enabled` | 1 | Use memory pool in *vt* for message allocation | +| `vt_zoltan_enabled` | 0 | Build with Zoltan enabled for `ZoltanLB` support | +| `vt_mpi_guards` | 0 | Guards against mis-use of MPI calls in code using *vt* | +| `vt_fcontext_enabled` | 0 | Enable user-level threads through boost fcontext | +| `vt_priorities_enabled` | 1 | Enable prioritization of work (adds bits in envelope) | +| `vt_priority_bits_per_level` | 3 | Number of bits per level of priority in envelope | +| `CODE_COVERAGE` | 0 | Enable code coverage for VT examples/tests | +| `VT_BUILD_TESTS` | 1 | Build all VT tests | +| `VT_BUILD_EXAMPLES` | 1 | Build all VT examples | + + +\subsection using-the-build-script Using the Build Script + +Instead of running `cmake`, one may invoke the `vt/ci/build_cpp.sh` script which +will run `cmake` for *vt* with environment variables for most configuration +parameters. + +\subsubsection building-environment-variables Build Script Environment Variables + +| Variable | Default Value | Description | +| ------------------ | --------------- | ----------- | +| `CMAKE_BUILD_TYPE` | Release | The `cmake` build type | +| `VT_LB_ENABLED` | 1 | Compile with support for runtime load balancing | +| `VT_TRACE_ENABLED ` | 0 | Compile with support for runtime tracing (Projections-format) | +| `VT_TRACE_RUNTIME_ENABLED ` | 0 | Force tracing on at runtime (used in CI for automatically testing tracing on all tests/examples) | +| `VT_DOXYGEN_ENABLED ` | 0 | Enable doxygen generation | +| `VT_MIMALLOC_ENABLED ` | 0 | Enable `mimalloc`, alternative allocator for debugging memory usage/frees/corruption | +| `VT_ASAN_ENABLED ` | 0 | Enable building with address sanitizer | +| `VT_POOL_ENABLED ` | 1 | Use memory pool in *vt* for message allocation | +| `VT_ZOLTAN_ENABLED ` | 0 | Build with Zoltan enabled for `ZoltanLB` support | +| `ZOLTAN_DIR ` | | Directory pointing to Zoltan installation | +| `VT_MPI_GUARD_ENABLED ` | 0 | Guards against mis-use of MPI calls in code using *vt* | + +With these set, invoke the script with two arguments: the path to the *vt* root +directory and the build path. Here's an example assuming that *vt* is cloned +into `/usr/src/vt` with trace enabled in debug mode. + +**Usage for building:** + +```bash +$ vt/ci/build_cpp.sh +``` + +**Example:** + +```bash +$ cd /usr/src +$ git clone git@github.com:DARMA-tasking/vt +$ VT_TRACE_ENABLED=1 CMAKE_BUILD_TYPE=Debug /usr/src/vt/ci/build_cpp.sh /usr/src/vt /usr/build/vt +``` + +\subsection docker-build Building with `docker` containerization + +The easiest way to build *vt* is by using `docker` with the available containers +that contain the proper compilers, MPI, and all other dependencies. First, +install `docker` on the system. On some systems, `docker-compose` might also +need to be installed. + +The `docker` builds are configured through `docker-compose` to use a shared, +cached filesystem mount with the host for `ccache` to enable fast re-builds. + +For `docker-compose`, the following variables can be set to configure the +build. One may configure the architecture, compiler type and version, Linux +distro (ubuntu or alpine), and distro version. + +The default set of the docker configuration options is located in `vt/.env`, +which `docker-compose` will read. + +``` +# Variables: +# ARCH={amd64, arm64v8, ...} +# COMPILER_TYPE={gnu, clang} +# COMPILER={gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10, +# clang-3.9, clang-4.0, clang-5.0, clang-6.0, clang-7, clang-8, +# clang-9, clang-10} +# REPO=lifflander1/vt +# UBUNTU={18.04, 20.04} +# ULIMIT_CORE=0 +# +# DARMA/vt Configuration Variables: +# VT_LB=1 # Enable load balancing +# VT_TRACE=0 # Enable tracing +# VT_MIMALLOC=0 # Enable mimalloc memory allocator +# VT_DOCS=0 # Enable doxygen build +# VT_TRACE_RT=0 # Enable tracing at runtime (for testing) +# VT_ASAN=0 # Enable address sanitizer +# BUILD_TYPE=release # CMake build type +``` + +With these set, run the following for a non-interactive build with ubuntu: + +```bash +$ cd vt +$ docker-compose run -e BUILD_TYPE=debug -e VT_TRACE=1 ubuntu-cpp +``` + +Or, alternatively, run a non-interactive build with alpine: + +```bash +$ cd vt +$ docker-compose run -e BUILD_TYPE=debug -e VT_TRACE=1 alpine-cpp +``` + +For an interactive build with ubuntu, where one can build, debug, and run +`valgrind`, etc: + +```bash +$ cd vt +$ docker-compose run -e BUILD_TYPE=debug -e VT_TRACE=1 ubuntu-cpp-interactive +# /vt/ci/build_cpp.sh /vt /build +# /vt/ci/test_cpp.sh /vt /build +``` + +The same call applies to alpine distro builds if you swap +`ubuntu-cpp-interactive` for `alpine-cpp-interactive`. + +For more detailed information on configuring the docker build, read the +documentation in `vt/docker-compose.yml`. + +\section test-vt Testing + +After *vt* is built succesfully, one may invoke the tests several ways. One may +run `make test` or `ninja test` (depending on the generator used) or `ctest`, to +run all the tests. Alternatively, the tests can be run automatically from the CI +script: + +```bash +$ vt/ci/test_cpp.sh +``` diff --git a/docs/md/collection.md b/docs/md/collection.md new file mode 100644 index 0000000000..1f6197d808 --- /dev/null +++ b/docs/md/collection.md @@ -0,0 +1,28 @@ +\page collection Virtual Context Collection +\brief Collection of tasks + +The virtual context collection component +`vt::vrt::collection::CollectionManager`, accessed via `vt::theCollection()` is +a core VT component that manages multi-dimensional collections of *virtual +context* (or a migratable C++ object registered with \vt) elements. It manages +the creation, deletion, and messaging across elements at runtime supporting +dense, sparse, on-demand, and staged insert modes. It utilizes the \subpage +location to manage the location of these elements to efficiently deliver +messages. It also utilizes the \subpage group to build a spanning tree across +the nodes that the collection is currently mapped to. This group makes +broadcasts efficient and allows reductions to make progress without waiting for +nodes that do not have collection elements. The \subpage proc-stats component +stores the statistics for live collections that then passes the instrumented +data to the \subpage lb-manager component to apply load balancing strategies. + +\section rooted-hello-world-collection Hello World 1D Dense Collection (Rooted) +\snippet examples/hello_world/hello_world_collection.cc Hello world collection + +\section collective-hello-world-collection Hello World 1D Dense Collection (Collective) +\snippet examples/hello_world/hello_world_collection_collective.cc Hello world collective collection + +\section reduce-hello-world-collection Hello World 1D Collection Reduce +\snippet examples/hello_world/hello_world_collection_reduce.cc Hello world reduce collection + +\section staged-insert-hello-world-collection Hello World 1D Collection Staged Insert +\snippet examples/hello_world/hello_world_collection_staged_insert.cc Hello world staged insert collection diff --git a/docs/md/collective.md b/docs/md/collective.md new file mode 100644 index 0000000000..2f43253e5b --- /dev/null +++ b/docs/md/collective.md @@ -0,0 +1,64 @@ +\page collective Collectives +\brief Collective operations + +The collective component `vt::collective::CollectiveAlg`, accessed via +`vt::theCollective()` implements active-message-based distributed collectives +over the \vt runtime. It performs asynchronous reductions, scatters, barriers, +and allows one to safely use MPI interspersed through \vt code, while running in +a handler. + +\section collective-reductions Reductions + +\vt comes with several reduction operators built in that can operate over types +where the operator already applies (like integers, doubles, etc.) or +user-defined overloaded operators. These operators also have specializations for +`std::vector` and `std::array`. + +| Operator | Reference | +| ------------- | -------------------------- | +| operator&& | `vt::collective::AndOp` | +| operator\|\| | `vt::collective::OrOp` | +| operator+ | `vt::collective::PlusOp` | +| std::min | `vt::collective::MinOp` | +| std::max | `vt::collective::MaxOp` | +| operator& | `vt::collective::BitAndOp` | +| operator\| | `vt::collective::BitOrOp` | +| operator^ | `vt::collective::BitXorOp` | +| | `vt::collective::NoneOp` | + +\subsection collective-reduce-example A Simple Reduction + +\code{.cpp} +#include + +// Reduce ints +struct ReduceDataMsg : vt::collective::ReduceTMsg { + explicit ReduceDataMsg(int val) + : vt::collective::ReduceTMsg(val) + { } +}; + +// Handler to target for reduction +struct ReduceResult { + void operator()(ReduceDataMsg* msg) { + auto num_nodes = vt::theContext()->getNumNodes(); + auto output = msg->getConstVal(); + fmt::print("reduction value={}\n", output); + vtAssert(num_nodes * 50 == output, "Must be equal); + } +}; + +int main(int argc, char** argv) { + vt::initialize(argc, argv); + + auto reduce_msg = vt::makeMessage(50); + + NodeType const root_reduce_node = 0; + vt::theCollective()->global()->reduce,ReduceResult>( + root_reduce_node, reduce_msg.get() + ); + + vt::finalize(); // spins in scheduler until termination + return 0; +} +\endcode diff --git a/docs/md/context.md b/docs/md/context.md new file mode 100644 index 0000000000..a7f97260f8 --- /dev/null +++ b/docs/md/context.md @@ -0,0 +1,36 @@ + +\page context Context +\brief Node-aware context + +The context component `vt::ctx::Context`, accessed via `vt::theContext()`, +provides context-aware querying of the current node (analogous to MPI's \c +MPI_Comm_rank), number of nodes (analogous to MPI's \c MPI_Comm_size), and +kernel threading/ULT information if worker threads are enabled. The context also +provides the MPI communicator that an instance of \vt is currently using. + +\copybrief vt::ctx::Context +\copydetails vt::ctx::Context + +\subsection get-node Current node/rank + +\copybrief vt::ctx::Context::getNode() + +To get the current node, one may query this method: + +\code{.cpp} +vt::NodeType this_node = vt::theContext()->getNode(); +\endcode + +\subsection get-num-nodes Number of nodes/ranks + +\copybrief vt::ctx::Context::getNumNodes() + +To get the number of nodes or ranks that an instance of \vt is using, one may +query this method: + +\code{.cpp} +vt::NodeType num_nodes = vt::theContext()->getNumNodes(); +\endcode + +\note The result from \c getNode or \c getNumNodes will depend on the +communicator that was passed to VT during initialization. diff --git a/docs/md/event.md b/docs/md/event.md new file mode 100644 index 0000000000..68a2ab36bf --- /dev/null +++ b/docs/md/event.md @@ -0,0 +1,10 @@ +\page event Event Manager +\brief Manage asynchronous events + +The async event component `vt::event::AsyncEvent`, accessed via `vt::theEvent()` +manages local and remote events that complete asynchronously. One may create an +event for a `MPI_Request` so the scheduler tests the event as the scheduler +polls. Once may also create other general events that have a unique ID that can +be tested remotely. Parent events group sets of other events (parent, normal, or +MPI events) together to test them for completion in a single operation. The +event manager is mostly designed for internal \vt usage. diff --git a/docs/md/examples.md b/docs/md/examples.md new file mode 100644 index 0000000000..f4219f36e8 --- /dev/null +++ b/docs/md/examples.md @@ -0,0 +1,8 @@ +\page examples Examples +\brief Small example programs to learn \vt + +| Example | Description | Link | +| ---------- | ----------- | ---- | +| Jacobi1D | 1-D Jacobi linear system solver with homogeneous Dirichlet condition | \subpage jacobi1d-example | +| Jacobi2D | 2-D Jacobi linear system solver with homogeneous Dirichlet condition | \subpage jacobi2d-example | +| ReduceIntegral | Composite trapezoidal rule for the integral `\int_{0}^{1} f(x) dx` | \subpage reduce-integral-example | diff --git a/docs/md/group.md b/docs/md/group.md new file mode 100644 index 0000000000..8e3fc10dd2 --- /dev/null +++ b/docs/md/group.md @@ -0,0 +1,19 @@ +\page group Group Manager +\brief Create a grouping of nodes + +The group manager component `vt::group::GroupManager`, accessed via +`vt::theGroup()` manages both rooted and collective groups (or subsets) of nodes +that can be broadcast to or reduced over. The group manager implements a fully +distributed algorithm for constructing groups collectively with each node +deciding if it should be included. The group manager builds a reasonably +balanced distributed spanning tree based on these collective votes. + +One major use case for the group manager is creating spanning trees for +reductions over virtual collections that do not span all the nodes. + +When creating a group, one may ask \vt to create a underlying MPI group, which +can be accessed once the group has finished construction. + +\section collective-group-example Example creating a collective group + +\snippet examples/group/group_collective.cc Collective group creation diff --git a/docs/md/jacobi1d-example.md b/docs/md/jacobi1d-example.md new file mode 100644 index 0000000000..d98c84ce50 --- /dev/null +++ b/docs/md/jacobi1d-example.md @@ -0,0 +1,6 @@ +\page jacobi1d-example Example Jacobi 1D Program + +The full code for this example can be found here: +`examples/collection/jacobi1d_vt.cc`. + +\snippet examples/collection/jacobi1d_vt.cc Jacobi1D example diff --git a/docs/md/jacobi2d-example.md b/docs/md/jacobi2d-example.md new file mode 100644 index 0000000000..16e48cc941 --- /dev/null +++ b/docs/md/jacobi2d-example.md @@ -0,0 +1,6 @@ +\page jacobi2d-example Example Jacobi 2D Program + +The full code for this example can be found here: +`examples/collection/jacobi2d_vt.cc`. + +\snippet examples/collection/jacobi2d_vt.cc Jacobi2D example diff --git a/docs/md/lb-manager.md b/docs/md/lb-manager.md new file mode 100644 index 0000000000..f097f8268e --- /dev/null +++ b/docs/md/lb-manager.md @@ -0,0 +1,59 @@ +\page lb-manager LB Manager +\brief Manage load balancers + +The LB manager component `vt::vrt::collection::balance::LBManager`, accessed via +`vt::theLBManager()` manages and coordinates instances of load balancers. It +counts collections as they call `nextPhase` to ensure they are all ready before +load balancing begins. It reads the command-line arguments or LB specification +file to determine which load balancer to run. + +To enable load balancing, the cmake flag \code{.cmake} -Dvt_lb_enabled=1 +\endcode should be passed during building. This also enables automatic +instrumentation of work and communication performed by collection elements. + +To run a load balancer at runtime: + - Pass `--vt_lb --vt_lb_name=` as a command line argument + - Write a LB specification file `--vt_lb --vt_lb_file --vt_lb_file_name=` + +\section lb-specification-file LB Specification File + +The LB specification file allows users to specify which load balancer along with +which LB-specific configuration parameters are passed to the load balancer +instance for any given phase. The order of the LB phase specficiation lines in +the file disambiguates lines---higher precedence for earlier lines. + +The format of the LB specification file is: + +\code +[%] <$phase> <$lbname> [$LB-specific-arg-1] ... [$LB-specfic-arg-N] +\endcode + +If a `%` is present, the line matches phases where: +`current phase % $phase == 0`. Phase-specific lines (ones that specify a load +balancer without a `%`) always always have precedence over `%` lines. The next +token after the optional `%` and `$phase` is the name of the load balancer to +invoke on that phase. After the load balancer name, `N` arguments to the load +balancer are allowed to customize how the load balancer is run with the format +of `key=value`. These arguments are the equivalent of passing +`--vt_lb_args="A=test B=test2"` on the command line. + +The following is an example LB specification: + +\code +%10 GossipLB c=1 k=5 f=2 i=10 +0 HierarchicalLB min=0.9 max=1.1 auto=false +% 5 GreedyLB min=1.0 +120 GreedyLB c=0 k=2 f=3 i=3 +\endcode + +\section load-balancers Load balancers + +| Load Balancer | Type | Description | Reference | +| -------------- | ----------------------- | ---------------------------------------------- | --------- | +| RotateLB | Testing | Rotate objects in a ring | `vt::vrt::collection::lb::RotateLB` | +| RandomLB | Testing | Randomly migrate object with seed | `vt::vrt::collection::lb::RandomLB` | +| GreedyLB | Centralized | Gather to central node apply min/max heap | `vt::vrt::collection::lb::GreedyLB` | +| GossipLB | Distributed | Gossip-based protocol for fully distributed LB | `vt::vrt::collection::lb::GossipLB` | +| HierarchicalLB | Hierarchical | Build tree to move objects nodes | `vt::vrt::collection::lb::HierarchicalLB` | +| ZotltanLB | Hyper-graph Partitioner | Run Zoltan in hyper-graph mode to LB | `vt::vrt::collection::lb::ZoltanLB` | +| StatsMapLB | User-specified | Read file to determine mapping | `vt::vrt::collection::lb::StatsMapLB` | diff --git a/docs/md/learn.md b/docs/md/learn.md new file mode 100644 index 0000000000..ab45a62022 --- /dev/null +++ b/docs/md/learn.md @@ -0,0 +1,10 @@ +\page learn Learn + +To learn how to use \vt, there are several resources that will help. + + - Learn \subpage vt-build + - Walk through the \subpage tutorial + - Read the small example programs + - Read the \subpage examples walk-through + - Examine `vt/examples/*` + - Read about the different \ref vt-components diff --git a/docs/md/location.md b/docs/md/location.md new file mode 100644 index 0000000000..d9d57915db --- /dev/null +++ b/docs/md/location.md @@ -0,0 +1,38 @@ +\page location Location Manager +\brief Virtual entity location management + +The location manager component `vt::location::LocationManager`, accessed via +`vt::theLocMan()` manages the location of arbitrary virtual entities in the +system. It holds a set of live `vt::location::EntityLocationCoord` across the +distributed system which allow users to register/unregister entities and inform +the system when migrations occur. With the entities registered, the location +coordinator can route messages to them even in the presence of migrations that +may occur at any time. The location coordinator maintains a cache of locations +for entities registered off-node and forwards messages using a communication +protocol that depends on the size of the message. + +Every entity in the system has a "home node", which is the node that is +ultimately responsible for knowing the location of the entity. When an entity +migrates, it informs the home node of its new location in the system. Nodes that +try to route messages to that entity will inquire the home node unless the +location is already in cache. + +\section comm-protocol Eager vs. Rendezvous Routing Protocol + +The variable `vt::location::small_msg_max_size` controls whether a message is +routed with an eager or rendezvous protocol. If the message is under that size +limit, the message is routed eagerly---forwarded to the "home node" for +resolution if the location is not in the cache. + +If the size of the message is greater than `vt::location::small_msg_max_size`, +the location coordinator will inquire with a control message to resolve the +location before the large message is actually sent. This reduces the number of +hops required to send large messages. + +\section location-migrations Entity Migrations + +When migrations occur at any time, it's always possible for the message to +arrive on a node where the entity *used to be*. In this case, the location +coordinator knows to follow the breadcrumb to get the message delivered properly +where the entity exists now. If the entity continues to move, the message will +"chase" it until it catches up. diff --git a/docs/md/mainpage.md b/docs/md/mainpage.md new file mode 100644 index 0000000000..270b84eb70 --- /dev/null +++ b/docs/md/mainpage.md @@ -0,0 +1,38 @@ +\mainpage DARMA + +*Efficient, distributed task-based programming made easy* + +Welcome to the DARMA documentation! + +The DARMA project is positioned at the forefront of asynchronous many-task (AMT) +research and development (R&D), providing production-quality dynamic runtime +software for scientific HPC applications. DARMA provides powerful capabilities +for dynamically scheduling tasks, incrementally (re-)balancing loads, data +serialization & movement, and programming abstractions that yield performance +portability. The DARMA team partners with key application/domain-specific teams +to drive development, implement requisite functionality, and production-harden +runtime software tailored to Sandia’s mission-critical applications. + +DARMA has two organizations on Github: one for the public facing code and +another for private/internal code, which has not be formally released. + + - DARMA Public Organization: [DARMA-tasking](https://github.com/DARMA-tasking) + - DARMA Private/Internal Organization: [DARMA-tasking-internal](https://github.com/DARMA-tasking-internal) + +The DARMA teams develops an integrated toolkit of codes/repositories that work +together to provide efficient, task-based distributed processing on large +supercomputer architectures. The main public repositories are: + +| Module | Name | Links | +| ---------------------------------- | -------------------------------------------------------------- | -------------------------- | +| HPC Runtime | @m_span{m-text m-success} DARMA/vt @m_endspan (Virtual Transport) | [Github](https://github.com/DARMA-tasking/vt) | +| HPC Serialization | @m_span{m-text m-success} DARMA/checkpoint @m_endspan (Checkpointing and Serialization Library) | [Github](https://github.com/DARMA-tasking/checkpoint) | +| C++ trait detection and checking | @m_span{m-text m-success} DARMA/detector @m_endspan (C++ trait detector) | [Github](https://github.com/DARMA-tasking/detector) | +| HPC LB Simulator | @m_span{m-text m-success} DARMA/LBAF @m_endspan (Load Balancing Analysis Framework) | [Github](https://github.com/DARMA-tasking/LB-analysis-framework) | +| HPC Serializer compiler analyzer | @m_span{m-text m-success} DARMA/checkpoint-analyzer @m_endspan (Static verification of serializers) | [Github](https://github.com/DARMA-tasking/checkpoint-member-analyzer) | +| Toolkit documentation | @m_span{m-text m-success} DARMA/docs @m_endspan | [Docs](https://github.com/DARMA-tasking/DARMA-tasking.github.io) | + +\section intro-darma-vt Learn about DARMA/vt + +DARMA/vt is the main runtime in the toolkit that provides tasking +functionality. To learn more about \vt, read the \subpage introduction. diff --git a/docs/md/memory-usage.md b/docs/md/memory-usage.md new file mode 100644 index 0000000000..a4dcfb549e --- /dev/null +++ b/docs/md/memory-usage.md @@ -0,0 +1,10 @@ +\page mem-usage Memory Usage Tracker +\brief Track memory usage + +The memory usage component `vt::util::memory::MemoryUsage`, accessed via +`vt::theMemUsage()` is an optional VT component that tracks memory usage over +time. It can be used with the \subpage trace component to write memory usage to +Projections log files to track usage after each registered function executes. It +can be configured to report usage after each LB phase is reached. This component +is backed by a wide range of different reporters---everything from trapping +memory allocation calls to counting allocated pages. diff --git a/docs/md/migrate-collection-example.md b/docs/md/migrate-collection-example.md new file mode 100644 index 0000000000..43795270ca --- /dev/null +++ b/docs/md/migrate-collection-example.md @@ -0,0 +1,6 @@ +\page migrate-collection-example Example Collection Migration Program + +The full code for this example can be found here: +`examples/collection/migrate_collection.cc`. + +\snippet examples/collection/migrate_collection.cc Migrate collection example diff --git a/docs/md/objgroup.md b/docs/md/objgroup.md new file mode 100644 index 0000000000..3bfd23cfbd --- /dev/null +++ b/docs/md/objgroup.md @@ -0,0 +1,11 @@ +\page objgroup Object Group Manager +\brief Create object instances across nodes + +The object group manager component `vt::objgroup::ObjGroupManager`, accessed via +`vt::theObjGroup()` allows the creation and management of instances of a group +of objects (one per node) that have a collective proxy for performing operations +like sends, broadcasts, or reductions across the object group. + +\section objgroup-example Example creating an object group + +\snippet examples/hello_world/objgroup.cc Object group creation diff --git a/docs/md/param.md b/docs/md/param.md new file mode 100644 index 0000000000..e51af1a698 --- /dev/null +++ b/docs/md/param.md @@ -0,0 +1,8 @@ +\page param Parameterization +\brief Handler parameterization + +@m_class{m-label m-danger} **Experimental** + +The parameterization component `vt::param::Param`, accessed via `vt::theParam()` +is an experimental component for parameterizing arguments into active function +handlers as an alternative to messages. diff --git a/docs/md/pipe.md b/docs/md/pipe.md new file mode 100644 index 0000000000..3e4baa8ee4 --- /dev/null +++ b/docs/md/pipe.md @@ -0,0 +1,25 @@ +\page pipe Pipe Manager +\brief Create opaque callback endpoints + +The pipe manager component `vt::pipe::PipeManager`, accessed via `vt::theCB()` +allows the creation of general pipes and callbacks between opaque endpoints that +are not revealed through the type. Callbacks allow one to supply a general +endpoint that accepts a type of data without revealing the actual endpoint +instance. For example, one may create a callback that triggers a handler +invocation on a certain node, broadcasts to a handler, sends to a collection or +objgroup, or broadcasts to a collection or objgroup, etc. + +The pipe manager supports more complex use cases of multi-listener endpoints if +one wants to trigger multiple endpoints on potentially different nodes. The +lifetime of a pipe can also be configured---how many invocations are allowed +before the callback is invalid. The pipe manager has a reference count for each +pipe which gets decremented with each signal arrival. By default, callbacks are +infinitely callable and do not expire. + +The pipe manager also supports "typed" callbacks where the callee type is +revealed to the caller. Typed callbacks are slightly more efficient because the +type is exposed and registered type-erasure is not required (using lambdas). + +\section callback-example Example callbacks + +\snippet examples/callback/callback.cc Callback examples diff --git a/docs/md/pool.md b/docs/md/pool.md new file mode 100644 index 0000000000..4e75dd9f48 --- /dev/null +++ b/docs/md/pool.md @@ -0,0 +1,14 @@ +\page pool Memory Pool +\brief Memory pool for efficient allocation + +The memory pool component `vt::pool::Pool`, accessed via `vt::thePool()` +provides a highly efficient memory pool for fixed sized allocations in three +sizes: small (`vt::pool::memory_size_small`), medium +(`vt::pool::memory_size_medium`), and large (currently unimplemented). + +All message allocation (on the send and receive side) is overloaded with +new/delete overloads to allocate message memory through the \vt memory pool. The +pool implementation uses a non-thread-safe allocation policy (must be +allocated/deallocated on the same thread) with fixed sized buckets. If the size +exceeds the largest bucket, the memory pool will fall back on the standard +allocator. diff --git a/docs/md/proc-stats.md b/docs/md/proc-stats.md new file mode 100644 index 0000000000..c6885edee7 --- /dev/null +++ b/docs/md/proc-stats.md @@ -0,0 +1,15 @@ +\page proc-stats Processor Statistics +\brief Manager object profiling data + +The processor statistics manager component +`vt::vrt::collection::balance::ProcStats`, accessed via `vt::theProcStats()` +manages instrumentation data from objects in a collection. It holds data per +node on the timing of these objects and communication between them demarcated by +phase and subphase. + +When LB is invoked in \vt, the \subpage lb-manager passes the processor +statistics to the various LB strategies to run the load balancer. The processor +statistics component can also dump the statistic data it holds to files, which +can be read externally. The LBAF (Load Balancing Analysis Framework) can also +then read this data to analyze the quality of the load distribution at any phase +in the file. diff --git a/docs/md/rdma.md b/docs/md/rdma.md new file mode 100644 index 0000000000..a5f823f699 --- /dev/null +++ b/docs/md/rdma.md @@ -0,0 +1,13 @@ +\page rdma RDMA Manager +\brief Node-level RDMA + +@m_class{m-label m-danger} **Experimental** + +The RDMA manager component `vt::rdma::RDMAManager`, accessed via `vt::theRDMA()` +is an experimental component that sends pure data to registered RDMA handlers or +directly to memory locations. + +Registered RDMA handlers trigger a function when the data arrives (GET) or is +sent (PUT). If registered memory locations are used directly, one may create a +RDMA channel which backs the GET/PUT by `MPI_Get`/`MPI_Put`. + diff --git a/docs/md/rdmahandle.md b/docs/md/rdmahandle.md new file mode 100644 index 0000000000..9590baea5a --- /dev/null +++ b/docs/md/rdmahandle.md @@ -0,0 +1,11 @@ +\page rdmahandle RDMA Handle Manager +\brief RDMA handles backed by MPI + +The RDMA handle manager component `vt::rdma::Manager`, accessed via +`vt::theHandleRDMA()` is a component that allows data to be transferred between +RDMA handles, which are persistent objects with underlying memory registered +with MPI. + +RDMA handles can either be node- or index-level, depending on whether they +belong to an objgroup or collection. A handle provides an interface to calling +get/put/accum to access the backing MPI implementation. diff --git a/docs/md/reduce-integral-example.md b/docs/md/reduce-integral-example.md new file mode 100644 index 0000000000..3095da49b7 --- /dev/null +++ b/docs/md/reduce-integral-example.md @@ -0,0 +1,6 @@ +\page reduce-integral-example Example Integral Reduction Program + +The full code for this example can be found here: +`examples/collection/reduce_integral.cc`. + +\snippet examples/collection/reduce_integral.cc Reduce integral example diff --git a/docs/md/registry.md b/docs/md/registry.md new file mode 100644 index 0000000000..59106b70f7 --- /dev/null +++ b/docs/md/registry.md @@ -0,0 +1,13 @@ +\page registry Registry +\brief Registered handlers + +The registry component `vt::registry::Registry`, accessed via +`vt::theRegistry()` holds type-safe active handlers for execution across a +distributed machine. + + - The \subpage active-messenger uses the registry to store/dispatch active +function and active functor handlers. + - The \subpage objgroup uses the registry to store/dispatch active member + functions + - The \subpage collection uses the registry to store/dispatch active functions + with the object pointer and active members. diff --git a/docs/md/scheduler.md b/docs/md/scheduler.md new file mode 100644 index 0000000000..72c87c11af --- /dev/null +++ b/docs/md/scheduler.md @@ -0,0 +1,55 @@ +\page scheduler Scheduler +\brief General scheduling of work + +The scheduler component `vt::sched::Scheduler`, accessed via `vt::theSched()` +holds pieces of work to execute later that may be prioritized. The scheduler +polls the \vt components to make progress and collect new pieces of work. The +scheduler allows registration of callbacks when the system is idle. + +\section calls-to-the-scheduler Calls to the scheduler + +To advance the scheduler, one can call it as follows: + +\code{.cpp} +vt::theSched()->scheduler(); +\endcode + +This polls every component that might generate or complete work, and potentially +runs one piece of available work. + +\copydoc vt::sched::Scheduler::scheduler(bool) + +However, if the scheduler needs to be run until a condition (or set of +conditions) is met, it is recommended that `runSchedulerWhile` be invoked: + +\copydoc vt::sched::Scheduler::runSchedulerWhile(std::function) + +\section higher-level-calls Higher-level Calls to Wait for Completion + +If work is enclosed in an "epoch", the \subpage term can be used to track its +distributed completion. In this case, instead of calling the scheduler directly, +built-in higher-level functions can be used to advance the scheduler until this +work is complete/terminated. + +To run the scheduler until an epoch terminates, call the following function: + +\code{.cpp} +vt::runSchedulerThrough(my_epoch); +\endcode + +Or, to combine the actual enclosed work with the call to wait for its +termination, use the following function: + +\code{.cpp} +vt::runInEpochRooted([]{ + // work to do on a single node +}); +\endcode + +If the work should be executed by all nodes, use a collective epoch:: + +\code{.cpp} +vt::runInEpochCollective([]{ + // work to do on all nodes +}); +\endcode diff --git a/docs/md/seq.md b/docs/md/seq.md new file mode 100644 index 0000000000..bbccc6c576 --- /dev/null +++ b/docs/md/seq.md @@ -0,0 +1,8 @@ +\page seq Sequencer +\brief Sequence node actions + +@m_class{m-label m-danger} **Experimental** + +The sequencer component `vt::seq::TaggedSequencer`, accessed via `vt::theSeq()` +orders operations on a node. If multiple handlers arrive, one can specify if +they are allowed to run in parallel and in what correct orders that may execute. diff --git a/docs/md/stats-reader.md b/docs/md/stats-reader.md new file mode 100644 index 0000000000..5795316f0f --- /dev/null +++ b/docs/md/stats-reader.md @@ -0,0 +1,17 @@ +\page stats-reader LB Restart Reader +\brief Follow input LB distribution + +The LB stats restart reader component +`vt::vrt::collection::balance::StatsRestartReader`, accessed via +`vt::theStatsReader()` reads in an input object distribution for a given program +and follows the distribution as specified in the file. + +A common flow is the following: + - Run the program to output stats files (with the flag `--vt_lb_stats`) + - Input those files to the LBAF (Load Balancing Analysis Framework) to generate a +new load distribution offline (e.g., to test a new LB strategy). + - Tell LBAF to generate a new set of stats files that contains a new mapping + of object to processor + - Run the program with the `StatsRestartReader` to test this new mapping on + the actual application + - Using the options `--vt_lb_stats_dir_in=inputdir --vt_lb_stats_file_in=filename` diff --git a/docs/md/term.md b/docs/md/term.md new file mode 100644 index 0000000000..3c39804aaa --- /dev/null +++ b/docs/md/term.md @@ -0,0 +1,32 @@ +\page term Termination Detector +\brief Detect termination of work + +The termination component `vt::term::TerminationDetector`, accessed via +`vt::theTerm()` detects the completion of the transitive closure of work by +following the causal chain of messages/events across multiple nodes. It provides +global termination to determine when all work is complete and the schedulers can +stop running. Additionally, it enables the creation of epochs (which stamp +message envelopes) to mark messages as part of a work grouping to detect +termination of all events causally related to a subset of messages in the +system. + +The termination detector comes with two different detection algorithms: (1) +4-counter wave-based termination for large collective or large rooted epochs +across the whole system; and, (2) Dijkstra-Scholten parental responsibility +termination for rooted epochs. Epochs are allowed to have other epochs nested +within them, thus forming a graph. The detector tracks the relation between +epochs, only making progress on epochs that do not have a dependency on another +epoch terminating first. + +The termination detector also comes with hang detection to detect causes where +no progress can be made due to bugs in an application's code or the runtime +implementation. When a hang is detected, if configured as such by the user, the +detector will dump a DOT graph of the live epochs and their dependencies. + +\section term-collective-example Example of creating a collective epoch + +\snippet examples/termination/termination_collective.cc Collective termination example + +\section term-rooted-example Example of creating a rooted epoch + +\snippet examples/termination/termination_rooted.cc Rooted termination example diff --git a/docs/md/trace.md b/docs/md/trace.md new file mode 100644 index 0000000000..377d18c724 --- /dev/null +++ b/docs/md/trace.md @@ -0,0 +1,64 @@ +\page trace Tracing +\brief Trace distributed events + +The optional trace component `vt::trace::Trace`, accessed via `vt::theTrace()` +builds a distributed trace of events, including VT handlers, user events, and +MPI invocations via the PMPI interface. It outputs +[Projections](http://charm.cs.uiuc.edu/software) log and sts files to enable +performance analysis after execution. + +To enable tracing at runtime, the trace component must be enabled at compile +time with cmake. To enable tracing pass the cmake flag: +\code{.cmake} +-Dvt_trace_enabled=1 +\endcode + +\section tracing-spec-file Tracing Specification File + +In order to customize when tracing is enabled and disabled, a trace +specification file can be passed to \vt via a command-line flag: +`--vt_trace_spec --vt_trace_spec_file=filename.spec`. + +The parser will read the following format: + +\code +[%] +\endcode + +The following is an example of a trace specification: + +\code +0 0 10 +%100 -3 3 +200 -5 5 +\endcode + +This specifies that tracing will be enabled on the following phases: + +\code +{ + [0,10], # phase 0 with offsets 0,+10 (subsumes [0,3] from %100 -3 3) + [97,103] # any phase % 100 with offset -3,+3 + [195,205] # phase 200 with offsets -5,+5 (subsumes [197,203] from %100 -3 3) + [297,303] # any phase % 100 with offset -3,+3 + [n%100-3,n%100+3] ... # any phase % 100 with offset -3,+3 +} +\endcode + +The sets of mod-phase and phase-specific entries must be unique. There may be +overlap across the two sets, but not within them. Having two entries that +start with `%100` or two entries that start with `100` would be invalid and +trigger a parsing error. But having a `%100` and `100` entry is valid. +Whether tracing is enabled is calculated as an OR across all specification +entries. Thus, if a given phase is contained in any spec line, it is +enabled. Note that `0 % 100 = 0`. Therefore, if the above example did not +contain the first line, tracing would be enabled as: + +\code +{ + [0,3], # any phase mod 100 from -3,+3 + [97,103], + [195,205], + [297,303], ... +} +\endcode diff --git a/docs/md/tutorial-1a.md b/docs/md/tutorial-1a.md new file mode 100644 index 0000000000..06eb077ace --- /dev/null +++ b/docs/md/tutorial-1a.md @@ -0,0 +1,4 @@ +\page tutorial-1a Learning about context + +\snippet tutorial/tutorial_1a.h Tutorial1A + diff --git a/docs/md/tutorial-1b.md b/docs/md/tutorial-1b.md new file mode 100644 index 0000000000..38999c57df --- /dev/null +++ b/docs/md/tutorial-1b.md @@ -0,0 +1,4 @@ +\page tutorial-1b Learning about active handlers + +\snippet tutorial/tutorial_1b.h Tutorial1B + diff --git a/docs/md/tutorial-1c.md b/docs/md/tutorial-1c.md new file mode 100644 index 0000000000..21f5002cd8 --- /dev/null +++ b/docs/md/tutorial-1c.md @@ -0,0 +1,4 @@ +\page tutorial-1c Learning about serialization + +\snippet tutorial/tutorial_1c.h Tutorial1C + diff --git a/docs/md/tutorial-1d.md b/docs/md/tutorial-1d.md new file mode 100644 index 0000000000..1cf816db73 --- /dev/null +++ b/docs/md/tutorial-1d.md @@ -0,0 +1,4 @@ +\page tutorial-1d Learning about broadcasts + +\snippet tutorial/tutorial_1d.h Tutorial1D + diff --git a/docs/md/tutorial-1e.md b/docs/md/tutorial-1e.md new file mode 100644 index 0000000000..e9a758f59a --- /dev/null +++ b/docs/md/tutorial-1e.md @@ -0,0 +1,4 @@ +\page tutorial-1e Learning about rooted groups + +\snippet tutorial/tutorial_1e.h Tutorial1E + diff --git a/docs/md/tutorial-1f.md b/docs/md/tutorial-1f.md new file mode 100644 index 0000000000..1966fc3b38 --- /dev/null +++ b/docs/md/tutorial-1f.md @@ -0,0 +1,4 @@ +\page tutorial-1f Learning about collective groups + +\snippet tutorial/tutorial_1f.h Tutorial1F + diff --git a/docs/md/tutorial-1g.md b/docs/md/tutorial-1g.md new file mode 100644 index 0000000000..3d18147447 --- /dev/null +++ b/docs/md/tutorial-1g.md @@ -0,0 +1,4 @@ +\page tutorial-1g Learning about callbacks + +\snippet tutorial/tutorial_1g.h Tutorial1G + diff --git a/docs/md/tutorial-1h.md b/docs/md/tutorial-1h.md new file mode 100644 index 0000000000..a9d2165300 --- /dev/null +++ b/docs/md/tutorial-1h.md @@ -0,0 +1,4 @@ +\page tutorial-1h Learning about reductions + +\snippet tutorial/tutorial_1h.h Tutorial1H + diff --git a/docs/md/tutorial-2a.md b/docs/md/tutorial-2a.md new file mode 100644 index 0000000000..8332fbb19e --- /dev/null +++ b/docs/md/tutorial-2a.md @@ -0,0 +1,4 @@ +\page tutorial-2a Learning about collections + +\snippet tutorial/tutorial_2a.h Tutorial2A + diff --git a/docs/md/tutorial-2b.md b/docs/md/tutorial-2b.md new file mode 100644 index 0000000000..80c98b8a36 --- /dev/null +++ b/docs/md/tutorial-2b.md @@ -0,0 +1,4 @@ +\page tutorial-2b Learning about collection reductions + +\snippet tutorial/tutorial_2b.h Tutorial2B + diff --git a/docs/md/tutorial-3a.md b/docs/md/tutorial-3a.md new file mode 100644 index 0000000000..93e3a9eecc --- /dev/null +++ b/docs/md/tutorial-3a.md @@ -0,0 +1,4 @@ +\page tutorial-3a Learning about collective epochs + +\snippet tutorial/tutorial_3a.h Tutorial3A + diff --git a/docs/md/tutorial.md b/docs/md/tutorial.md new file mode 100644 index 0000000000..e0b042a30d --- /dev/null +++ b/docs/md/tutorial.md @@ -0,0 +1,54 @@ +\page tutorial Tutorial +\brief Tutorial of how to use \vt + +\section tutorial-init-finalize-vt Initializing/Finalizing + +To initialize \vt all you must do is invoke the `vt::initialize` call with the +program arguments, which creates and initializes a new \vt runtime. The +initialize call reads the \vt arguments and removes them, leaving the remaining +for the program. + +\code{.cpp} +int main(int argc, char** argv) { + vt::initialize(argc, argv); + // program here + vt::finalize(); +} +\endcode + +If you are already using MPI in your program already, you don't want to call +initialize like this because \vt will initialize MPI itself. Instead, you should +pass a pointer to the live MPI communicator to \vt for it to clone, as so: + +\code{.cpp} +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + vt::initialize(argc, argv, &MPI_COMM_WORLD); + // program here + vt::finalize(); + MPI_Finalize(); +} +\endcode + +\section tutorial-walkthrough Tutorial Code Snippets + +This page walks through the tutorial that exists in the source code. See +`vt/tutorial/*.h` for the tutorial pieces in the repository. The main tutorial +example code that stitches the pieces together is located in example: +`vt/tutorial/tutorial_main.h`. The actual code that compiles into an example is +in `vt/examples/tutorial.cc`. + + - Tutorial 1---Basic Concepts + - 1a: \subpage tutorial-1a + - 1b: \subpage tutorial-1b + - 1c: \subpage tutorial-1c + - 1d: \subpage tutorial-1d + - 1e: \subpage tutorial-1e + - 1f: \subpage tutorial-1f + - 1g: \subpage tutorial-1g + - 1h: \subpage tutorial-1h + - Tutorial 2---Collections + - 2a: \subpage tutorial-2a + - 2b: \subpage tutorial-2b + - Tutorial 3---Epochs + - 3a: \subpage tutorial-3a diff --git a/docs/md/vrtseq.md b/docs/md/vrtseq.md new file mode 100644 index 0000000000..63c0a60d3b --- /dev/null +++ b/docs/md/vrtseq.md @@ -0,0 +1,9 @@ +\page vrtseq Virtual Sequencer +\brief Sequence task actions + +@m_class{m-label m-danger} **Experimental** + +The sequencer component `vt::seq::TaggedSequencerVrt`, accessed via +`vt::theVirtualSeq()` orders operations on a virtual context. If multiple +handlers arrive, one can specify if they are allowed to run in parallel and in +what correct orders that may execute. diff --git a/docs/md/vt.md b/docs/md/vt.md new file mode 100644 index 0000000000..d0b2e7c21f --- /dev/null +++ b/docs/md/vt.md @@ -0,0 +1,164 @@ +\page introduction Introduction +\brief Overview of functionality in \vt + +\tableofcontents + +\section what-is What is vt? + +\vt is an active messaging layer that utilizes C++ object virtualization to +manage virtual endpoints with automatic location management. \vt is directly +built on top of MPI to provide efficient portability across different machine +architectures. Empowered with virtualization, \vt can automatically perform +dynamic load balancing to schedule scientific applications across diverse +platforms with minimal user input. + +\vt abstracts the concept of a `node`/`rank`/`worker`/`thread` so a program can +be written in terms of virtual entities that are location independent. Thus, +they can be automatically migrated and thereby executed on varying hardware +resources without explicit programmer mapping, location, and communication +management. + +\section vt-features Features in vt + + - Active messaging to type-safe handlers across nodes + - Groups for scalable construction of node subsets + - Optional serialization of messages + - Termination detection across all work or subsets of work with \e epochs + - Opaque callbacks/pipes to generalized endpoints + - Efficient memory pooling for message allocation + - RDMA using MPI one-sided for data transfer + - Asynchronous Collectives across nodes/groups (scatter, async barrier, reduce, ...) + - General scheduler with prioritization + - Built-in interoperability with MPI and threading libraries (Kokkos, OpenMP, ...) + - Object groups for node-level encapsulation + - Virtual contexts for migratable virtualization and dispatch + - Abstractions for multi-dimensional indices, mapping, and linearization + - Virtual collections (dense, sparse, dynamic insertable) for decomposing domains + - Fully distributed load balancer for virtual entities + +\section vt-components Components in vt + +| Component | Singleton | Details | Type | +| --------------------------- | ---------------------- | --------------------------- | ---------------------------------------------- | +| \subpage context | `vt::theContext()` | \copybrief context | @m_class{m-label m-success} **Core** | +| \subpage active-messenger | `vt::theMsg()` | \copybrief active-messenger | @m_class{m-label m-success} **Core** | +| \subpage collection | `vt::theCollection()` | \copybrief collection | @m_class{m-label m-success} **Core** | +| \subpage collective | `vt::theCollective()` | \copybrief collective | @m_class{m-label m-success} **Core** | +| \subpage event | `vt::theEvent()` | \copybrief event | @m_class{m-label m-success} **Core** | +| \subpage group | `vt::theGroup()` | \copybrief group | @m_class{m-label m-success} **Core** | +| \subpage lb-manager | `vt::theLBManager()` | \copybrief lb-manager | @m_class{m-label m-warning} **Optional** | +| \subpage location | `vt::theLocMan()` | \copybrief location | @m_class{m-label m-success} **Core** | +| \subpage mem-usage | `vt::theMemUsage()` | \copybrief mem-usage | @m_class{m-label m-warning} **Optional** | +| \subpage objgroup | `vt::theObjGroup()` | \copybrief objgroup | @m_class{m-label m-success} **Core** | +| \subpage param | `vt::theParam()` | \copybrief param | @m_class{m-label m-danger} **Experimental** | +| \subpage pipe | `vt::theCB()` | \copybrief pipe | @m_class{m-label m-success} **Core** | +| \subpage proc-stats | `vt::theProcStats()` | \copybrief proc-stats | @m_class{m-label m-warning} **Optional** | +| \subpage pool | `vt::thePool()` | \copybrief pool | @m_class{m-label m-success} **Core** | +| \subpage rdma | `vt::theRDMA()` | \copybrief rdma | @m_class{m-label m-danger} **Experimental** | +| \subpage rdmahandle | `vt::theHandleRDMA()` | \copybrief rdmahandle | @m_class{m-label m-warning} **Optional** | +| \subpage registry | `vt::theRegistry()` | \copybrief registry | @m_class{m-label m-success} **Core** | +| \subpage scheduler | `vt::theSched()` | \copybrief scheduler | @m_class{m-label m-success} **Core** | +| \subpage seq | `vt::theSeq()` | \copybrief seq | @m_class{m-label m-danger} **Experimental** | +| \subpage vrtseq | `vt::theVirtualSeq()` | \copybrief vrtseq | @m_class{m-label m-danger} **Experimental** | +| \subpage term | `vt::theTerm()` | \copybrief term | @m_class{m-label m-success} **Core** | +| \subpage trace | `vt::theTrace()` | \copybrief trace | @m_class{m-label m-warning} **Optional** | +| \subpage stats-reader | `vt::theStatsReader()` | \copybrief stats-reader | @m_class{m-label m-warning} **Optional** | + + +\section how-to-build-intro How to build + +\vt can be built with cmake or inside a docker container. Learn \subpage +vt-build + +\section vt-hello-world Hello World Example + +\m_class{m-block m-success} +\parblock + \m_class{m-code-figure} \parblock + \code{.cpp} + bool done = false; + + struct HelloMsg : vt::Message { + HelloMsg(vt::NodeType in_from) : from(in_from) { } + vt::NodeType from = 0; + }; + + void hello_world(HelloMsg* msg) { + vt::NodeType this_node = vt::theContext()->getNode(); + fmt::print("{}: Hello from node {}\n", this_node, msg->from); + done = true; + } + + int main(int argc, char** argv) { + vt::initialize(argc, arv); + + vt::NodeType this_node = vt::theContext()->getNode(); + vt::NodeType num_nodes = vt::theContext()->getNumNodes(); + + if (this_node == 0) { + auto msg = vt::makeMessage(this_node); + vt::theMsg()->broadcastMsg(msg.get()); + done = true; + } + + // Run the scheduler until all nodes are done + vt::runSchedulerWhile([]{ return !done; }); + + vt::finalize(); + return 0; + } + \endcode + + Running: + + \code{.shell-session} + $ mpirun -n 4 ./hello_world + \endcode + + Output: + \code{.shell-session} + 3: Hello from node 0 + 1: Hello from node 0 + 2: Hello from node 0 + \endcode + + \note An active message broadcast sends to all nodes except for +the sender (root of the broadcast). + \endparblock +\endparblock + +\section License + +@m_class{m-note m-dim} + +@parblock +Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC +(NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +Government retains certain rights in this software. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +@endparblock diff --git a/examples/callback/callback.cc b/examples/callback/callback.cc index d717a7d1b1..a689a8fd52 100644 --- a/examples/callback/callback.cc +++ b/examples/callback/callback.cc @@ -44,6 +44,7 @@ #include +/// [Callback examples] // Message sent from the callback to the callback endpoint struct TestMsg : vt::Message { using MessageParentType = ::vt::Message; @@ -161,3 +162,4 @@ int main(int argc, char** argv) { return 0; } +/// [Callback examples] diff --git a/examples/collection/jacobi1d_vt.cc b/examples/collection/jacobi1d_vt.cc index 9f70779d9a..eea942116b 100644 --- a/examples/collection/jacobi1d_vt.cc +++ b/examples/collection/jacobi1d_vt.cc @@ -42,6 +42,7 @@ //@HEADER */ +/// [Jacobi1D example] // // This code applies a few steps of the Jacobi iteration to @@ -383,4 +384,4 @@ int main(int argc, char** argv) { return 0; } - +/// [Jacobi1D example] diff --git a/examples/collection/jacobi2d_vt.cc b/examples/collection/jacobi2d_vt.cc index ed64bc801c..5f800999db 100644 --- a/examples/collection/jacobi2d_vt.cc +++ b/examples/collection/jacobi2d_vt.cc @@ -48,6 +48,7 @@ #include #include +/// [Jacobi2D example] // // This code applies a few steps of the Jacobi iteration to @@ -516,3 +517,4 @@ int main(int argc, char** argv) { return 0; } +/// [Jacobi2D example] diff --git a/examples/collection/migrate_collection.cc b/examples/collection/migrate_collection.cc index caa31a3f40..4ff1a8d98f 100644 --- a/examples/collection/migrate_collection.cc +++ b/examples/collection/migrate_collection.cc @@ -44,6 +44,7 @@ #include +/// [Migrate collection example] static constexpr int32_t const default_num_elms = 16; struct Hello : vt::Collection { @@ -125,3 +126,4 @@ int main(int argc, char** argv) { return 0; } +/// [Migrate collection example] diff --git a/examples/collection/reduce_integral.cc b/examples/collection/reduce_integral.cc index abaa711461..f693fff685 100644 --- a/examples/collection/reduce_integral.cc +++ b/examples/collection/reduce_integral.cc @@ -42,6 +42,7 @@ //@HEADER */ +/// [Reduce integral example] // // This code computes a composite trapezoidal rule for the integral @@ -249,4 +250,4 @@ int main(int argc, char** argv) { return 0; } - +/// [Reduce integral example] diff --git a/examples/group/group_collective.cc b/examples/group/group_collective.cc index a4d46f2f7b..6dc8191746 100644 --- a/examples/group/group_collective.cc +++ b/examples/group/group_collective.cc @@ -44,14 +44,7 @@ #include -struct HelloMsg : vt::Message { - int from; - - explicit HelloMsg(int const& in_from) - : from(in_from) - { } -}; - +/// [Collective group creation] struct HelloGroupMsg : vt::Message { }; static void hello_group_handler(HelloGroupMsg* msg) { @@ -110,3 +103,4 @@ int main(int argc, char** argv) { return 0; } +/// [Collective group creation] diff --git a/examples/hello_world/hello_world_collection.cc b/examples/hello_world/hello_world_collection.cc index 8c0406e607..9bb1cfd004 100644 --- a/examples/hello_world/hello_world_collection.cc +++ b/examples/hello_world/hello_world_collection.cc @@ -44,6 +44,7 @@ #include +/// [Hello world collection] struct Hello : vt::Collection { Hello() = default; @@ -83,3 +84,4 @@ int main(int argc, char** argv) { return 0; } +/// [Hello world collection] diff --git a/examples/hello_world/hello_world_collection_collective.cc b/examples/hello_world/hello_world_collection_collective.cc index da889148d0..0429990822 100644 --- a/examples/hello_world/hello_world_collection_collective.cc +++ b/examples/hello_world/hello_world_collection_collective.cc @@ -44,6 +44,7 @@ #include +/// [Hello world collective collection] struct Hello : vt::Collection { Hello() = default; @@ -83,3 +84,4 @@ int main(int argc, char** argv) { return 0; } +/// [Hello world collective collection] diff --git a/examples/hello_world/hello_world_collection_reduce.cc b/examples/hello_world/hello_world_collection_reduce.cc index eaac7ef29b..eeea94f410 100644 --- a/examples/hello_world/hello_world_collection_reduce.cc +++ b/examples/hello_world/hello_world_collection_reduce.cc @@ -44,6 +44,7 @@ #include +/// [Hello world reduce collection] struct Hello : vt::Collection { using ReduceMsg = vt::collective::ReduceTMsg; @@ -88,3 +89,4 @@ int main(int argc, char** argv) { return 0; } +/// [Hello world reduce collection] diff --git a/examples/hello_world/hello_world_collection_staged_insert.cc b/examples/hello_world/hello_world_collection_staged_insert.cc index a0941004d4..31af5676d1 100644 --- a/examples/hello_world/hello_world_collection_staged_insert.cc +++ b/examples/hello_world/hello_world_collection_staged_insert.cc @@ -44,6 +44,7 @@ #include +/// [Hello world staged insert collection] struct Hello : vt::Collection { // Default constructor for migration @@ -107,3 +108,4 @@ int main(int argc, char** argv) { return 0; } +/// [Hello world staged insert collection] diff --git a/examples/hello_world/objgroup.cc b/examples/hello_world/objgroup.cc index 99a33d8029..b0fa72fca0 100644 --- a/examples/hello_world/objgroup.cc +++ b/examples/hello_world/objgroup.cc @@ -44,6 +44,7 @@ #include +/// [Object group creation] struct MyMsg : vt::Message { MyMsg(int in_a, int in_b) : a(in_a), b(in_b) { } int a = 0, b = 0; @@ -76,3 +77,4 @@ int main(int argc, char** argv) { return 0; } +/// [Object group creation] diff --git a/examples/termination/termination_collective.cc b/examples/termination/termination_collective.cc index 5ff5ad33d9..b3454e87d3 100644 --- a/examples/termination/termination_collective.cc +++ b/examples/termination/termination_collective.cc @@ -44,6 +44,7 @@ #include "vt/transport.h" +/// [Collective termination example] using TestMsg = vt::Message; vt::NodeType nextNode() { @@ -98,3 +99,4 @@ int main(int argc, char** argv) { return 0; } +/// [Collective termination example] diff --git a/examples/termination/termination_rooted.cc b/examples/termination/termination_rooted.cc index 905a9be918..828052815e 100644 --- a/examples/termination/termination_rooted.cc +++ b/examples/termination/termination_rooted.cc @@ -44,6 +44,7 @@ #include "vt/transport.h" +/// [Rooted termination example] using TestMsg = vt::Message; vt::NodeType nextNode() { @@ -96,4 +97,4 @@ int main(int argc, char** argv) { return 0; } - +/// [Rooted termination example] diff --git a/src/vt/collective/barrier/barrier.h b/src/vt/collective/barrier/barrier.h index 42e67006f2..115a5f4bdd 100644 --- a/src/vt/collective/barrier/barrier.h +++ b/src/vt/collective/barrier/barrier.h @@ -59,60 +59,179 @@ namespace vt { namespace collective { namespace barrier { constexpr BarrierType const fst_barrier = 1; constexpr BarrierType const fst_coll_barrier = 0x8000000000000001; +/** + * \struct Barrier + * + * \brief Perform a collective barrier that is safe to use with VT handlers in + * flight. + * + * Align execution across multiple nodes to ensure each node across the + * communicator/runtime reach a "matching" barrier. + * + * \warning Barriers are not recommended for users for parallel + * coordination. First, barriers do not guarantee that work is actually done + * when the barrier is reached. It only ensures that the node reaches that + * point. Thus, messages sent or work enqueued for the scheduler might not be + * done when a barrier is reached. For ensuring that work is complete, use the + * \c TerminationDetector component to create an epoch that groups the piece of + * work (see \c vt::runInEpochRooted and \c vt::runInEpochCollective ). Second, + * barriers may limit the concurrency in a program; in many cases only a + * reduction is necessary for correctness. + */ struct Barrier : virtual collective::tree::Tree { using BarrierStateType = BarrierState; + /** + * \internal \brief Construct a new barrier manager + */ Barrier(); template using ContainerType = std::unordered_map; + /** + * \internal \brief Insert/find a barrier + * + * \param[in] is_named whether the barrier is named + * \param[in] is_wait whether the barrier is of waiting type + * \param[in] barrier the barrier ID + * \param[in] cont_action (optional) continuation to attach after completion + * + * \return the barrier state + */ BarrierStateType& insertFindBarrier( bool const& is_named, bool const& is_wait, BarrierType const& barrier, ActionType cont_action = nullptr ); + /** + * \internal \brief Remove the state of a barrier + * + * \param[in] is_named whether the barrier is named + * \param[in] is_wait whether the barrier is of waiting type + * \param[in] barrier the barrier ID + */ void removeBarrier( bool const& is_named, bool const& is_wait, BarrierType const& barrier ); + /** + * \brief Rooted call to create a new named barrier, returning the ID. + * + * After calling this, one must broadcast/send this to other nodes. Typical + * use case is to put it in a message for later coordination. + * + * \return the barrier ID + */ BarrierType newNamedBarrier(); + + /** + * \brief Collective call to create a new named barrier, returning the ID. + * + * All nodes must call this in the same order to generate a consistent barrier + * ID for waiting on later. + * + * \return the barrier ID + */ BarrierType newNamedCollectiveBarrier(); + /** + * \internal \brief Send a barrier up the tree + * + * \param[in] is_named whether the barrier is named + * \param[in] is_wait whether the barrier is of waiting type + * \param[in] barrier the barrier ID + * \param[in] skip_term whether to skip termination (mark barrier a TD message) + */ void barrierUp( bool const& is_named, bool const& is_wait, BarrierType const& barrier, bool const& skip_term ); + /** + * \internal \brief Send a barrier down the tree to release nodes (barrier is + * reached!) + * + * \param[in] is_named whether the barrier is named + * \param[in] is_wait whether the barrier is of waiting type + * \param[in] barrier the barrier ID + */ void barrierDown( bool const& is_named, bool const& is_wait, BarrierType const& barrier ); + /** + * \brief Wait on a barrier + * + * \param[in] poll_action action to execute while polling for barrier + * completion in addition to polling scheduler + * \param[in] barrier the barrier ID to wait on + */ inline void barrier( ActionType poll_action = nullptr, BarrierType const& barrier = no_barrier ) { return waitBarrier(poll_action, barrier); } + /** + * \brief Collectively create a new barrier and once completed execute an + * action. + * + * \param[in] fn the action to execute after the barrier is reached by all + * nodes + */ inline void barrierThen(ActionType fn) { return contBarrier(fn); } + /** + * \brief Collective wait for a barrier and once completed execute an action. + * + * \param[in] barrier the barrier to wait on + * \param[in] fn the action to execute after the barrier is reached by all + * nodes + */ inline void barrierThen(BarrierType const& barrier, ActionType fn) { return contBarrier(fn, barrier); } + /** + * \internal \brief Collectively barrier skipping termination + * + * \warning This is dangerous to call in any code outside of internal + * initialize and finalize. + */ inline void systemMetaBarrier() { bool const skip_term = true; return waitBarrier(nullptr, no_barrier, skip_term); } + /** + * \internal \brief Collectively barrier skipping termination and then execute + * a continuation. + * + * \param[in] fn action to execute after barrier is reached + * + * \warning This is dangerous to call in any code outside of internal + * initialize and finalize. + */ inline void systemMetaBarrierCont(ActionType fn) { bool const skip_term = true; return contBarrier(fn, no_barrier, skip_term); } + /** + * \internal \brief Active handler to send a barrier up the spanning tree + * + * \param[in] msg the barrier message + */ static void barrierUp(BarrierMsg* msg); + + /** + * \internal \brief Active handler to send a barrier down the spanning tree + * + * \param[in] msg the barrier message + */ static void barrierDown(BarrierMsg* msg); private: diff --git a/src/vt/collective/collective_alg.h b/src/vt/collective/collective_alg.h index fa8d776e7a..73154e5eab 100644 --- a/src/vt/collective/collective_alg.h +++ b/src/vt/collective/collective_alg.h @@ -64,6 +64,15 @@ namespace vt { namespace collective { constexpr CollectiveAlgType const fst_collective_alg = 1; +/** + * \struct CollectiveAlg + * + * \brief Perform asynchronous collectives within VT + * + * CollectiveAlg is a core VT component that provides the ability to perform + * reductions, scatters, barriers, and safe MPI (collective) operations while + * inside a VT handler. + */ struct CollectiveAlg : runtime::component::Component, virtual reduce::ReduceManager, diff --git a/src/vt/collective/collective_scope.h b/src/vt/collective/collective_scope.h index e9fc2ffe5c..6ba8573ee7 100644 --- a/src/vt/collective/collective_scope.h +++ b/src/vt/collective/collective_scope.h @@ -82,6 +82,19 @@ struct ScopeImpl { } /* end namespace detail */ +/** + * \struct CollectiveScope + * + * \brief A distinct scope for enqueuing ordered collective operations. + * + * Each collective scope builds an individual sequence of collective operations + * that get orchestrated across nodes using a consensus algorithm so all nodes + * agree on a particular operation to execute in the sequence. + * + * An example use case is running a blocking MPI collective (like a + * \c * MPI_Allreduce) inside VT handlers without causing progress to halt due + * to asynchrony in VT. + */ struct CollectiveScope { private: diff --git a/src/vt/collective/reduce/reduce.h b/src/vt/collective/reduce/reduce.h index 3d0cb277b7..aeeebb3e54 100644 --- a/src/vt/collective/reduce/reduce.h +++ b/src/vt/collective/reduce/reduce.h @@ -67,18 +67,55 @@ namespace vt { namespace collective { namespace reduce { +/** + * \struct Reduce + * + * \brief A specific, isolated reducer instance for a given scope that sequences + * reduce operations via the reduction stamp within that scope. + * + * Holds the state as a reduction makes it up the spanning tree until it reaches + * the root. Combines messages with the user-specified reduction operator as + * they move up the spanning tree. + */ struct Reduce : virtual collective::tree::Tree { using ReduceStateType = ReduceState; using ReduceNumType = typename ReduceStateType::ReduceNumType; + /** + * \internal \brief Construct a new reducer instance + * + * \param[in] in_scope the scope for the reducer + */ explicit Reduce(detail::ReduceScope const& in_scope); + /** + * \internal \brief Construct a new reducer instance with a custom + * (or non-global) spanning tree + * + * \param[in] in_scope the scope for the reducer + * \param[in] in_tree the spanning tree + */ Reduce( detail::ReduceScope const& in_scope, collective::tree::Tree* in_tree ); + /** + * \internal \brief Generate the next reduction stamp + * + * \return the stamp + */ detail::ReduceStamp generateNextID(); + /** + * \brief Reduce a message up the tree + * + * \param[in] root the root node where the final handler provides the result + * \param[in] msg the message to reduce on this node + * \param[in] id the reduction stamp (optional), provided if out-of-order + * \param[in] num_contrib number of expected contributions from this node + * + * \return the next reduction stamp + */ template * f> detail::ReduceStamp reduce( NodeType root, MsgT* const msg, @@ -86,6 +123,17 @@ struct Reduce : virtual collective::tree::Tree { ReduceNumType num_contrib = 1 ); + /** + * \brief Reduce a message up the tree + * + * \param[in] root the root node where the final handler provides the result + * \param[in] msg the message to reduce on this node + * \param[in] cb the callback to trigger on the root node + * \param[in] id the reduction stamp (optional), provided if out-of-order + * \param[in] num_contrib number of expected contributions from this node + * + * \return the next reduction stamp + */ template < typename OpT, typename MsgT, @@ -99,6 +147,16 @@ struct Reduce : virtual collective::tree::Tree { ReduceNumType const& num_contrib = 1 ); + /** + * \brief Reduce a message up the tree with a target function on the root node + * + * \param[in] root the root node where the final handler provides the result + * \param[in] msg the message to reduce on this node + * \param[in] id the reduction stamp (optional), provided if out-of-order + * \param[in] num_contrib number of expected contributions from this node + * + * \return the next reduction stamp + */ template < typename OpT, typename FunctorT, @@ -111,31 +169,58 @@ struct Reduce : virtual collective::tree::Tree { ReduceNumType const& num_contrib = 1 ); + /** + * \internal \brief Combine in a new message for a given reduction + * + * \param[in] msg the message to combine with the operator + * \param[in] local was this called locally or from an incoming handler + * \param[in] num_contrib the number of expected local contributions + */ template void reduceAddMsg( MsgT* msg, bool const local, ReduceNumType num_contrib = -1 ); + /** + * \internal \brief Combine and send up the tree if ready + * + * \param[in] msg the message to reduce + */ template void reduceNewMsg(MsgT* msg); - /* - * Explicitly start the reduction when the number of contributions is not - * known up front + /** + * \internal \brief Explicitly start the reduction when the number of + * contributions is not known up front + * + * \param[in] id the reduce stamp to start + * \param[in] use_num_contrib whether to use the cached # of contributions */ template void startReduce(detail::ReduceStamp id, bool use_num_contrib = true); + /** + * \internal \brief Active function when a message reaches the root of the + * spanning tree and the reduction is complete + * + * \param[in] msg the reduce message + */ template void reduceRootRecv(MsgT* msg); + /** + * \internal \brief Active function when a message arrives for a given scope + * at some level in the spanning tree + * + * \param[in] msg the reduce message + */ template void reduceUp(MsgT* msg); private: - detail::ReduceScope scope_; - ReduceStateHolder state_; - detail::StrongSeq next_seq_; + detail::ReduceScope scope_; /**< The reduce scope for this reducer */ + ReduceStateHolder state_; /**< Reduce state, holds messages, etc. */ + detail::StrongSeq next_seq_; /**< The next reduce stamp */ }; }}} /* end namespace vt::collective::reduce */ diff --git a/src/vt/collective/reduce/reduce_manager.h b/src/vt/collective/reduce/reduce_manager.h index 332a753c19..58381ac1d9 100644 --- a/src/vt/collective/reduce/reduce_manager.h +++ b/src/vt/collective/reduce/reduce_manager.h @@ -52,37 +52,118 @@ namespace vt { namespace collective { namespace reduce { struct Reduce; +/** + * \struct ReduceManager + * + * \brief Manage distinct scopes for reductions + * + * Manages distinct reducers with an associated scope to orchestrate + * multiple asynchronous sequences of reduce operations. + */ struct ReduceManager { using ReducePtrType = std::unique_ptr; using ReduceScopeType = detail::ReduceScopeHolder; ReduceManager(); + /** + * \brief Get the global reducer + * + * \warning Using the global reducer is not recommended because it might + * conflict with other reductions in the global context. To create a new + * scope, one may call \c makeReducerCollective to collectively create a new + * reducer scope. + * + * \return the reducer + */ Reduce* global(); + /** + * \brief Get the reducer for a given scope + * + * \param[in] scope the scope + * + * \return the reducer + */ Reduce* getReducer(detail::ReduceScope const& scope); + /** + * \internal \brief Get the reducer for an objgroup + * + * \param[in] proxy the objgroup proxy + * + * \return the reducer + */ Reduce* getReducerObjGroup(ObjGroupProxyType const& proxy); + /** + * \internal \brief Get the reducer for a collection + * + * \param[in] proxy the collection proxy + * + * \return the reducer + */ Reduce* getReducerVrtProxy(VirtualProxyType const& proxy); + /** + * \internal \brief Get the reducer for a group + * + * \param[in] group the group ID + * + * \return the reducer + */ Reduce* getReducerGroup(GroupType const& group); + /** + * \internal \brief Get the reducer for a VT component + * + * Each VT component that inherits from \c runtime::component::Component + * gets its own scope. + * + * \param[in] cid the component ID + * + * \return the reducer + */ Reduce* getReducerComponent(ComponentIDType const& cid); + /** + * \brief Collectively make a new reducer that creates a unique reduction + * scope + * + * \return the reducer with a new scope + */ Reduce* makeReducerCollective(); + /** + * \internal \brief Create the reducer for a group when a custom (or + * non-global) spanning tree is required (cannot be created on demand) + * + * \param[in] group the group ID + * \param[in] tree the associated spanning tree for the group + */ void makeReducerGroup(GroupType const& group, collective::tree::Tree* tree); + /** + * \internal \brief Active function when a message reaches the root of the + * spanning tree and the reduction is complete + * + * \param[in] msg the reduce message + */ template static void reduceRootRecv(MsgT* msg); + /** + * \internal \brief Active function when a message arrives for a given scope + * at some level in the spanning tree + * + * \param[in] msg the reduce message + */ template static void reduceUp(MsgT* msg); private: - ReduceScopeType reducers_; - detail::UserIDType cur_user_id_ = 0; + ReduceScopeType reducers_; /**< Live reducers by scope */ + detail::UserIDType cur_user_id_ = 0; /**< The next user ID for a scope */ }; }}} /* end namespace vt::collective::reduce */ diff --git a/src/vt/collective/scatter/scatter.h b/src/vt/collective/scatter/scatter.h index 756f8e742b..0c6e297668 100644 --- a/src/vt/collective/scatter/scatter.h +++ b/src/vt/collective/scatter/scatter.h @@ -56,12 +56,34 @@ namespace vt { namespace collective { namespace scatter { +/** + * \struct Scatter + * + * \brief Scatter data across all nodes from a single origin + * + * Performs an asynchronous scatter over all the nodes in the + * communicator/runtime. + */ struct Scatter : virtual collective::tree::Tree { using FuncSizeType = std::function; using FuncDataType = std::function; + /** + * \internal \brief Construct a scatter manager + */ Scatter(); + /** + * \brief Scatter data to all nodes + * + * The functions passed to scatter through the arguments \c size_fn and + * \c data_fn will not be retained after this call returns. + * + * \param[in] total_size total size of data to scatter + * \param[in] max_proc_size max data to be scattered to any node + * \param[in] size_fn callback to get size for each node + * \param[in] data_fn callback to get data for each node + */ template * f> void scatter( std::size_t const& total_size, std::size_t const& max_proc_size, @@ -69,13 +91,35 @@ struct Scatter : virtual collective::tree::Tree { ); protected: + /** + * \internal \brief Receive scattered data down the spanning tree + * + * \param[in] msg the scatter message + */ void scatterIn(ScatterMsg* msg); private: + /** + * \internal \brief Helper function to scatter data + * + * \param[in] node the current code + * \param[in] ptr pointer to raw data + * \param[in] elm_size bytes per element of raw data + * \param[in] size_fn callback to get size for each node + * \param[in] data_fn callback to get data for each node + * + * \return incremented point after scatter is complete + */ char* applyScatterRecur( NodeType node, char* ptr, std::size_t elm_size, FuncSizeType size_fn, FuncDataType data_fn ); + + /** + * \internal \brief Active function to receive scattered data + * + * \param[in] msg the scatter message + */ static void scatterHandler(ScatterMsg* msg); }; diff --git a/src/vt/collective/tree/tree.h b/src/vt/collective/tree/tree.h index 75f1180d0d..bf249ecf0d 100644 --- a/src/vt/collective/tree/tree.h +++ b/src/vt/collective/tree/tree.h @@ -59,29 +59,131 @@ namespace vt { namespace collective { namespace tree { static struct DefaultTreeConstructTag { } tree_cons_tag_t { }; #pragma GCC diagnostic pop +/** + * \internal \struct Tree + * + * \brief General interface for storing a spanning tree + * + * Holds the portion of a spanning tree on each node. Does not do any parallel + * coordination. Higher-level components must keep track of the pieces of the + * tree on each node. + */ struct Tree { using NodeListType = std::vector; using OperationType = std::function; using NumLevelsType = int32_t; + /** + * \internal \brief Construct the default spanning tree across the whole + * communicator. + * + * \param[in] DefaultTreeConstructTag constructor tag + */ explicit Tree(DefaultTreeConstructTag); + + /** + * \internal \brief Construct a spanning tree with a list of children nodes on + * the root node + * + * \warning This should only be called on the root. + * + * \param[in] in_children the list of children + */ explicit Tree(NodeListType const& in_children); + /** + * \internal \brief Construct a spanning tree with a list of children nodes, + * parent node, and whether this is the root. + * + * \param[in] in_is_root if this is the root node + * \param[in] parent the node's parent in the tree + * \param[in] in_children the list of children + */ Tree( bool const in_is_root, NodeType const& parent, NodeListType const& in_children ); + /** + * \internal \brief Setup the default (binomial) tree + */ void setupTree(); + + /** + * \internal \brief Get the parent node in the tree + * + * \return the parent node + */ NodeType getParent() const; + + /** + * \internal \brief Get the number of children nodes + * + * \return the number of children nodes + */ NodeType getNumChildren() const; + + /** + * \internal \brief Get whether this node is the root + * + * \return whether this node is the root + */ bool isRoot() const; + + /** + * \internal \brief Get the (const) list of children + * + * \return (const) list of children + */ NodeListType const& getChildren() const; + + /** + * \internal \brief Get the list of children for a particular node in the tree + * + * \return list of children + */ NodeListType getChildren(NodeType node) const; + + /** + * \internal \brief Apply function (foreach) across all children + * + * \param[in] op action to apply, passed the node + */ void foreachChild(OperationType op) const; + + /** + * \internal \brief Get number of levels in the tree + * + * \return number of levels + */ NumLevelsType numLevels() const; + + /** + * \internal \brief Apply function (foreach) across all children with number + * of levels passed to apply function. + * + * \param[in] level number of levels in spanning tree + * \param[in] op action to apply, passed levels and node + */ void foreachChild(NumLevelsType level, OperationType op) const; + + /** + * \internal \brief Get total number of descendants for a particular node in + * the tree. + * + * The returned count does not include the node passed---only its descendants. + * + * \param[in] child the node + * + * \return number of descendants + */ std::size_t getNumTotalChildren(NodeType child) const; + + /** + * \internal \brief Get total children in tree + * + * \return number of children + */ std::size_t getNumTotalChildren() const; private: diff --git a/src/vt/event/event.h b/src/vt/event/event.h index 6d309d549d..a0020ff46f 100644 --- a/src/vt/event/event.h +++ b/src/vt/event/event.h @@ -63,12 +63,26 @@ namespace vt { namespace event { +/** \file */ + +/** + * \internal \brief State of an event that may be local or remote + * + * Tracks the state of a distributed event + */ enum class EventState : int8_t { - EventReady = 1, - EventWaiting = 2, - EventRemote = 3 + EventReady = 1, /**< Indicates event is ready/satisfied */ + EventWaiting = 2, /**< Indicates event is waiting on op */ + EventRemote = 3 /**< Indicates event is non-local */ }; +/** + * \struct AsyncEvent event.h vt/event/event.h + * + * \brief Used to track events + * + * Component to track events in the system to trigger actions or other events + */ struct AsyncEvent : runtime::component::PollableComponent { using EventRecordTypeType = eEventRecord; using EventManagerType = EventIDManager; @@ -87,13 +101,78 @@ struct AsyncEvent : runtime::component::PollableComponent { void initialize() override; void finalize() override; + + /** + * \brief Create a new event + * + * \param[in] type the type of event to create + * \param[in] node the node that's embedded in the event bit field + * + * \return a new event identifier + */ EventType createEvent(EventRecordTypeType const& type, NodeType const& node); + + /** + * \brief Get the event record + * + * \param[in] event the event identifier + * + * \return the event record + */ EventRecordType& getEvent(EventType const& event); + + /** + * \brief Get the owning node for an event + * + * \param[in] event the event identifier + * + * \return the node that owns the event + */ NodeType getOwningNode(EventType const& event); + + /** + * \brief Create a new MPI event that holds a MPI_Request + * + * \param[in] node the node on which the MPI event exists + * + * \return the event identifier + */ EventType createMPIEvent(NodeType const& node); + + /** + * \brief Create a regular type event + * + * \param[in] node the node that owns it + * + * \return the event identifier + */ EventType createNormalEvent(NodeType const& node); + + /** + * \brief Create a parent event that can have multiple children + * + * \param[in] node the node that owns it + * + * \return the event identifier + */ EventType createParentEvent(NodeType const& node); + + /** + * \brief Get the holder for an event + * + * \param[in] event the event identifier + * + * \return the holder + */ EventHolderType& getEventHolder(EventType const& event); + + /** + * \brief Check if a holder exist for an event + * + * \param[in] event the event identifier + * + * \return if it exists + */ bool holderExists(EventType const& event); bool needsPolling(EventRecordTypeType const& type); void removeEventID(EventType const& event); diff --git a/src/vt/group/group_manager.h b/src/vt/group/group_manager.h index 6420a17a0c..031f18550e 100644 --- a/src/vt/group/group_manager.h +++ b/src/vt/group/group_manager.h @@ -73,6 +73,16 @@ namespace vt { namespace group { +/** + * \struct GroupManager + * + * \brief A core VT component that can create and manage groups of nodes. + * + * Enables rooted and collective asynchronous construction of groups of + * nodes. The default group is all the nodes in the communicator, which is the + * group a normal broadcast targets. Beyond this, both the VT system and users + * can dynamically create groups in a fully distributed manner. + */ struct GroupManager : runtime::component::Component { using RegionType = region::Region; using RegionPtrType = std::unique_ptr; @@ -90,6 +100,9 @@ struct GroupManager : runtime::component::Component { using ReducePtrType = ReduceType*; using CollectiveScopeType = collective::CollectiveScope; + /** + * \internal \brief Construct the GroupManager + */ GroupManager(); virtual ~GroupManager() { @@ -101,20 +114,76 @@ struct GroupManager : runtime::component::Component { std::string name() override { return "GroupManager"; } + /** + * \internal \brief Setup the default group spanning all nodes. Called by the + * VT system at startup so broadcasts can take place. + */ void setupDefaultGroup(); + /** + * \brief Create a new rooted group. + * + * \param[in] in_region list of nodes to include + * \param[in] is_collective whether it's collective, which must be false + * \param[in] is_static whether the group is static after creation + * \param[in] action action to execute when group is finished construction + * + * \return the group ID + */ GroupType newGroup( RegionPtrType in_region, bool const& is_collective, bool const& is_static, ActionGroupType action ); + /** + * \brief Create a new rooted group + * + * \param[in] in_region list of nodes to include + * \param[in] action action to execute when group is finished construction + * + * \return the group ID + */ GroupType newGroup(RegionPtrType in_region, ActionGroupType action); + + /** + * \brief Create a group collectively, must be called on all nodes + * + * \param[in] in_group whether this node is included the new group + * \param[in] action action to execute when group is finished construction + * \param[in] make_mpi_group whether VT should create an underlying MPI group + * + * \return the group ID + */ GroupType newGroupCollective( bool const in_group, ActionGroupType action, bool make_mpi_group = false ); + + /** + * \internal \brief Generate the next group ID + * + * \param[in] GroupCollectiveLabelTagType tag to indicate collective + * + * \return the group ID + */ GroupType newGroupCollectiveLabel(GroupCollectiveLabelTagType); + + /** + * \brief Query whether this node is part of a group + * + * \param[in] group the group ID + * + * \return whether this node is included + */ bool inGroup(GroupType const& group); + /** + * \brief Get MPI_Comm from VT group + * + * \param[in] group_id the group ID + * + * \return the MPI_Comm associated with the group; returns \c MPI_COMM_WORLD + * if group was created without a communicator + */ MPI_Comm getGroupComm(GroupType const& group_id); template *f> @@ -127,40 +196,136 @@ struct GroupManager : runtime::component::Component { friend struct GroupActiveAttorney; private: + /** + * \internal \brief System call to create a new collective group + * + * \param[in] in_group whether the node is in the group + * \param[in] is_static whether the group is static after creation + * \param[in] action action to execute when group is finished construction + * \param[in] make_mpi_group whether VT should create an underlying MPI group + * + * \return the group ID + */ GroupType newCollectiveGroup( bool const& in_group, bool const& is_static, ActionGroupType action, bool make_mpi_group = false ); + + /** + * \internal \brief System call to create a new rooted group + * + * \param[in] in_region list of nodes in group + * \param[in] is_static whether the group is static after creation + * \param[in] action action to execute when group is finished construction + * + * \return the group ID + */ GroupType newLocalGroup( RegionPtrType in_region, bool const& is_static, ActionGroupType action ); + + /** + * \internal \brief Setup new collective group + * + * \param[in] group the group ID + * \param[in] is_static whether the group is static after creation + * \param[in] action action to execute when group is finished construction + * \param[in] in_group whether this node is included the new group + * \param[in] make_mpi_group whether VT should create an underlying MPI group + */ void initializeLocalGroupCollective( GroupType const& group, bool const& is_static, ActionType action, bool const in_group, bool make_mpi_group ); + + /** + * \internal \brief Setup a new rooted group locally + * + * \param[in] group the group ID + * \param[in] in_region list of nodes in group + * \param[in] is_static whether the group is static after creation + * \param[in] action action to execute when group is finished construction + * \param[in] group_size the number of nodes in the group + */ void initializeLocalGroup( GroupType const& group, RegionPtrType in_region, bool const& is_static, ActionType action, RegionType::SizeType const& group_size ); + + /** + * \internal \brief Setup a new rooted group remotely + * + * \param[in] group the group ID + * \param[in] in_region list of nodes in group + * \param[in] is_static whether the group is static after creation + * \param[in] group_size the number of nodes in the group + */ void initializeRemoteGroup( GroupType const& group, RegionPtrType in_region, bool const& is_static, RegionType::SizeType const& group_size ); + /** + * \internal \brief Get new collective identifier for action registration + * + * \return the operation ID + */ RemoteOperationIDType nextCollectiveID() { return cur_collective_id_++; } + /** + * \internal \brief Collectively register a operation ID continuation + * + * \param[in] action the continuation + * + * \return the operation ID + */ RemoteOperationIDType registerContinuation(ActionType action); + + /** + * \internal \brief Collectively register a operation ID continuation + * + * \param[in] op the operation ID + * \param[in] action the continuation + */ void registerContinuation( RemoteOperationIDType const& op, ActionType action ); + + /** + * \internal \brief Trigger a continuation with an operation ID + * + * \param[in] op the operation ID + */ void triggerContinuation(RemoteOperationIDType const& op); + /** + * \internal \brief Send message to a rooted group + * + * \param[in] base the message to send + * \param[in] from sender node + * \param[in] size number of bytes of the message being sent + * \param[in] is_root whether this node is the root + * \param[out] deliver whether the caller should deliver locally + * + * \return the event ID for any generated events (like MPI_Requests) + */ EventType sendGroup( MsgSharedPtr const& base, NodeType const& from, MsgSizeType const& size, bool const is_root, bool* const deliver ); + /** + * \internal \brief Send message to a collective group + * + * \param[in] base the message to send + * \param[in] from sender node + * \param[in] size number of bytes of the message being sent + * \param[in] is_root whether this node is the root + * \param[out] deliver whether the caller should deliver locally + * + * \return the event ID for any generated events (like MPI_Requests) + */ EventType sendGroupCollective( MsgSharedPtr const& base, NodeType const& from, MsgSizeType const& size, bool const is_root, @@ -168,14 +333,61 @@ struct GroupManager : runtime::component::Component { ); public: + /** + * \brief Get the reducer associated with a group to reduce over it + * + * \param[in] group the group ID + * + * \return pointer to the reducer + */ ReducePtrType groupReducer(GroupType const& group); + + /** + * \brief Get the root node for a group + * + * \param[in] group the group ID + * + * \return the root node + */ NodeType groupRoot(GroupType const& group) const; + + /** + * \brief Check if a group is the default group (all nodes, default spanning + * tree) + * + * \param[in] group the group ID + * + * \return whether it is the default group + */ bool groupDefault(GroupType const& group) const; + /** + * \internal \brief Add a cleanup action + * + * \param[in] action the action + */ void addCleanupAction(ActionType action); + + /** + * \internal \brief Get the next operation ID + * + * \return the operation ID + */ RemoteOperationIDType getNextID(); private: + /** + * \internal \brief Execute necessary actions for deliver to a group when a + * message arrives on this node + * + * \param[in] msg the message to send + * \param[in] from sender node + * \param[in] size number of bytes of the message being sent + * \param[in] is_root whether this node is the root + * \param[out] deliver whether the caller should deliver locally + * + * \return the event ID for any generated events (like MPI_Requests) + */ static EventType groupHandler( MsgSharedPtr const& msg, NodeType const& from, MsgSizeType const& msg_size, bool const is_root, @@ -195,9 +407,16 @@ struct GroupManager : runtime::component::Component { CollectiveScopeType collective_scope_; }; -// This is a separate template class because Intel 18 didn't like -// static members that were variable templates. These are all the -// members of GroupManager that were under a template +/** + * \internal \struct GroupManagerT + * + * \brief Group manager that handles typed actions that are registered for + * system use. + * + * This is a separate template class because Intel 18 didn't like static members + * that were variable templates. These are all the members of GroupManager that + * were under a template + */ template struct GroupManagerT : public GroupManager { diff --git a/src/vt/messaging/active.h b/src/vt/messaging/active.h index 67bd20b465..f9e804b91c 100644 --- a/src/vt/messaging/active.h +++ b/src/vt/messaging/active.h @@ -179,7 +179,6 @@ struct BufferedActiveMsg { }; /** - * \internal * \struct ActiveMessenger active.h vt/messaging/active.h * * \brief Core component of VT used to send messages. diff --git a/src/vt/objgroup/manager.h b/src/vt/objgroup/manager.h index ecf620e0ae..0214fb2293 100644 --- a/src/vt/objgroup/manager.h +++ b/src/vt/objgroup/manager.h @@ -66,6 +66,20 @@ namespace vt { namespace objgroup { +/** + * \struct ObjGroupManager + * + * \brief A core VT component that can create groups with one object per node + * + * Create a "object group" with one instance of that group on each node. An + * instantiated object group has once instance on each node that can be + * collectively referred to with a single proxy. This proxy allows + * sends/broadcasts/reduces over the object group. + * + * Object groups create a clean isolation of a instance of some functionality + * that has distributed behavior. Some of the newer VT components are + * implemented as object groups, such as the load balancers. + */ struct ObjGroupManager : runtime::component::Component { template using ProxyType = proxy::Proxy; @@ -80,6 +94,9 @@ struct ObjGroupManager : runtime::component::Component { using MsgContainerType = std::vector>; using BaseProxyListType = std::set; + /** + * \internal \brief Construct the ObjGroupManager + */ ObjGroupManager() = default; std::string name() override { return "ObjGroupManager"; } @@ -90,104 +107,251 @@ struct ObjGroupManager : runtime::component::Component { * communicator */ - // Make obj group with the constructor args to the obj + /** + * \brief Collectively construct a new object group. Allocates and constructs + * the object on each node by forwarding constructor arguments. + * + * \param[in] args args to pass to the object's constructor on each node + * + * \return proxy to the object group + */ template ProxyType makeCollective(Args&&... args); - // Make obj group with a std::unique_ptr to the obj + + /** + * \brief Collectively construct a new object group from a existing unique + * pointer to the local object + * + * \param[in] obj the std::unique_ptr to the local object + * + * \return proxy to the object group + */ template ProxyType makeCollective(std::unique_ptr obj); - // Make obj group with fn for creating the obj + + /** + * \brief Collectively construct a new object group with a callback to provide + * a unique pointer on each node. + * + * \param[in] fn callback function to construct + * + * \return proxy to the object group + */ template ProxyType makeCollective(MakeFnType fn); - // Make obj group with non-owning ptr to the object + + /** + * \brief Collectively construct a new object group from a raw pointer to the + * object. + * + * \warning This overload requires the caller to manage the lifetime of the + * object. Do not allow the object to be deallocated before the object group + * is destroyed. + * + * \param[in] obj raw pointer to the object + * + * \return proxy to the object group + */ template ProxyType makeCollective(ObjT* obj); - // Make obj group with specialized smart ptr (e.g., RCP, shared_ptr, etc.) + + /** + * \brief Collectively construct a new object group from a smart-pointer-like + * handle. + * + * \param[in] obj the smart-pointer-like handle that the system holds until + * destruction + * + * \return proxy to the object group + */ template