From fea96e1b2debdcbc896123b23cf642fb42dd6b3e Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Thu, 24 Aug 2023 12:04:11 -0500 Subject: [PATCH] docs: Better explain required versions of support libraries Get rid of the table that listed the required support library versions -- the table format was too limiting. Instead, add a bunch more verbiage about the versions that are included in the Open MPI distribution tarball and the minimum required versions of each of hwloc, libevent, PMIx, and PRRTE. Pay special attention to the corner cases of building PMIx (internal and external), with and without PRRTE. Also set the minimum required versions for PMIx and PRRTE in VERSIONS: * Testing shows that Open MPI v5.0.x requires at least PMIx v4.2.0 * The minimum required version for PRRTE is v3.0.0, but we recommend in the docs that users use >=v3.0.1 so that they get a full mpirun(1) man page Finally, also show in the docs the versions of the embedded packages (hwloc, libevent, PMIx, and PRRTE). This required adding a little Python in docs/conf.py to read VERSION files and extract version numbers from tarball filenames. Signed-off-by: Jeff Squyres --- VERSION | 4 +- docs/Makefile.am | 4 +- docs/conf.py | 108 ++++++++++---- .../required-support-libraries.rst | 141 ++++++++++++------ 4 files changed, 178 insertions(+), 79 deletions(-) diff --git a/VERSION b/VERSION index 2178439c11a..b2ba2da0c1f 100644 --- a/VERSION +++ b/VERSION @@ -25,8 +25,8 @@ mpi_standard_subversion=1 # OMPI required dependency versions. # List in x.y.z format. -pmix_min_version=4.1.2 -prte_min_version=2.0.2 +pmix_min_version=4.2.0 +prte_min_version=3.0.0 hwloc_min_version=1.11.0 event_min_version=2.0.21 automake_min_version=1.13.4 diff --git a/docs/Makefile.am b/docs/Makefile.am index dc9a085e99e..2023ece395b 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -1032,8 +1032,8 @@ $(ALL_MAN_BUILT): filename=`basename $$file`; \ cp -pf $(OMPI_PRRTE_RST_CONTENT_DIR)/$$filename "$(builddir)/prrte-rst-content"; \ done - $(OMPI_V_SPHINX_HTML) OMPI_VERSION_FILE=$(top_srcdir)/VERSION $(SPHINX_BUILD) -M html "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS) - $(OMPI_V_SPHINX_MAN) OMPI_VERSION_FILE=$(top_srcdir)/VERSION $(SPHINX_BUILD) -M man "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS) + $(OMPI_V_SPHINX_HTML) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M html "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS) + $(OMPI_V_SPHINX_MAN) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M man "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS) # A useful rule to invoke manually to ensure that all of the external # HTML links we have are valid. Running this rule requires diff --git a/docs/conf.py b/docs/conf.py index b8b7e8c4690..f0d0e283092 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,46 +14,85 @@ # -- Project information ----------------------------------------------------- +import os +import re import datetime + year = datetime.datetime.now().year project = 'Open MPI' copyright = f'2003-{year}, The Open MPI Community' author = 'The Open MPI Community' -# The full version, including alpha/beta/rc tags -# Read the Open MPI version from the VERSION file in the source tree -# The docs/Makefile.am will set the env var OMPI_VERSION_FILE, because -# we might be doing a VPATH build. -filename = None -if 'OMPI_VERSION_FILE' in os.environ: - filename = os.environ['OMPI_VERSION_FILE'] -elif os.path.exists("../VERSION"): - filename = '../VERSION' - -if filename is None: - print("ERROR: Could not find Open MPI source tree VERSION file") - exit(1) - -with open(filename) as fp: - ompi_lines = fp.readlines() - -ompi_data = dict() -for ompi_line in ompi_lines: - if '#' in ompi_line: - parts = ompi_line.split("#") - ompi_line = parts[0] - ompi_line = ompi_line.strip() - - if '=' not in ompi_line: - continue +# --------------------------- - ompi_key, ompi_val = ompi_line.split("=") - ompi_data[ompi_key.strip()] = ompi_val.strip() +# The docs/Makefile.am will set the env var OMPI_TOP_SRCDIR, because +# we might be doing a VPATH build. +ompi_top_srcdir = '..' +if 'OMPI_TOP_SRCDIR' in os.environ: + ompi_top_srcdir = os.environ['OMPI_TOP_SRCDIR'] + +# Read an Open MPI-style VERSION file +def read_version_file(path): + if not os.path.exists(path): + print(f"ERROR: Unable to find file {path}") + exit(1) + + with open(path) as fp: + version_lines = fp.readlines() + + data = dict() + for line in version_lines: + if '#' in line: + parts = line.split("#") + line = parts[0] + line = line.strip() + + if '=' not in line: + continue + + key, val = line.split("=") + data[key.strip()] = val.strip() + + return data + +# Look for a version string via a regular expresion of a filename in a +# given directory +def get_tarball_version(path, expr): + if not os.path.exists(path): + print(f"ERROR: Unable to find path {path}") + exit(1) + + for file in os.listdir(path): + m = re.match(expr, file) + if not m: + continue + return m.group(1) + + return "" + +# Read all the various versions from the source tree + +ompi_data = read_version_file(f"{ompi_top_srcdir}/VERSION") +pmix_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/openpmix/VERSION") +prte_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/prrte/VERSION") + +hwloc_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/", + r"hwloc-(.*).tar") +event_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/", + r"libevent-(.*)-stable.tar") + +# --------------------------- + +# Assemble several different combinations of version strings ompi_series = f"v{ompi_data['major']}.{ompi_data['minor']}.x" ompi_ver = f"v{ompi_data['major']}.{ompi_data['minor']}.{ompi_data['release']}{ompi_data['greek']}" +pmix_embedded_version = f"v{pmix_data['major']}.{pmix_data['minor']}.{pmix_data['release']}{pmix_data['greek']}" +prte_embedded_version = f"v{prte_data['major']}.{prte_data['minor']}.{prte_data['release']}{prte_data['greek']}" +prte_embedded_series = f"v{prte_data['major']}.{prte_data['minor']}" + pmix_min_version = f"{ompi_data['pmix_min_version']}" prte_min_version = f"{ompi_data['prte_min_version']}" hwloc_min_version = f"{ompi_data['hwloc_min_version']}" @@ -86,7 +125,6 @@ # If we're building in an RTD environment for a tag or external (i.e., # PR), use the RTD version -- not what we just read from the VERSIONS # file. -import os key = 'READTHEDOCS' if key in os.environ and os.environ[key] == 'True': print("OMPI: found ReadTheDocs build environment") @@ -172,9 +210,6 @@ # -- Options for MAN output ------------------------------------------------- -import os -import re - # Dynamically find all the man pages and build the appropriate list of # tuples so that we don't have to manually maintain it. @@ -222,9 +257,14 @@ def _doit(topdir): .. |ompi_ver| replace:: {ompi_ver} .. |ompi_series| replace:: {ompi_series} .. |pmix_min_version| replace:: {pmix_min_version} +.. |pmix_embedded_version| replace:: {pmix_embedded_version} .. |prte_min_version| replace:: {prte_min_version} +.. |prte_embedded_version| replace:: {prte_embedded_version} +.. |prte_embedded_series| replace:: {prte_embedded_series} .. |hwloc_min_version| replace:: {hwloc_min_version} +.. |hwloc_embedded_version| replace:: {hwloc_embedded_version} .. |event_min_version| replace:: {event_min_version} +.. |event_embedded_version| replace:: {event_embedded_version} .. |automake_min_version| replace:: {automake_min_version} .. |autoconf_min_version| replace:: {autoconf_min_version} .. |libtool_min_version| replace:: {libtool_min_version} @@ -234,6 +274,10 @@ def _doit(topdir): .. |mpi_standard_minor_version| replace:: {mpi_standard_minor_version} .. |deprecated_favor| replace:: this routine is deprecated in favor of +.. |br| raw:: html + +
+ """ # The sphinx_rtd_theme does not properly handle wrapping long lines in diff --git a/docs/installing-open-mpi/required-support-libraries.rst b/docs/installing-open-mpi/required-support-libraries.rst index b411e1a02f5..f88a7987fc1 100644 --- a/docs/installing-open-mpi/required-support-libraries.rst +++ b/docs/installing-open-mpi/required-support-libraries.rst @@ -3,50 +3,28 @@ Required support libraries ========================== -Open MPI requires the following support libraries with the minimum listed versions: - -.. list-table:: - :header-rows: 1 - - * - Library - - Minimum version - - Notes - * - `Hardware Locality `_ - - |hwloc_min_version| - - This library is required; Open MPI will not build without it. - * - `Libevent `_ - - |event_min_version| - - This library is required; Open MPI will not build without it. - * - `PMIx `_ - - |pmix_min_version| - - This library is required; Open MPI will not build without it. - * - `PRRTE `_ - - |prte_min_version| - - This library is optional in some environments. PRRTE provides - Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI - application launchers (the two are identical; they are symbolic - links to the same executable). - - * If your environment uses another MPI application launcher - (e.g., Slurm users can use the ``srun`` launcher to "direct - launch" Open MPI applications), then the use of PRRTE is - optional. - * If your environment has no other MPI application launcher, then - you need to install PRRTE and build Open MPI with PRRTE - support. - * Open MPI can use the copy of PRRTE embedded in its source - code tree, or compile/link against an external PRRTE - installation. :ref:`See this section for details about how - to specify each method - `. - -Since these support libraries are fundamental to Open MPI's operation -and not universally available in all environments, they are directly + +While Open MPI can be built with support for a wide variety of +systems, a small set of support libraries are *required* in order to +build Open MPI in *any* environment. Several of these packages are +both fundamental to Open MPI's operation and not universally available +in all environments. As such, these "fundamental" packages are both +embedded in Open MPI's distribution tarballs and also directly incorporated into Open MPI's configure, build, and installation -process. More on this below. +process. + +:ref:`See below +` for a +description of how Open MPI chooses whether to use the embedded +versions of these packages or versions already installed on your +system. + +* `Hardware Locality `_ - .. note:: The versions listed in this table are the *minimum* versions needed. In general, the Open MPI community recommends using more recent versions of both the :ref:`required support libraries ` and any other optional support libraries. This is because more recent versions typically tend to include bug fixes, sometimes affecting Open MPI functionality. As a specific example, there is a known issue with `Hardware Locality `_ releases older than v2.8.0 on systems with Intel Ponte Vecchio accelerators. If you run Open MPI on such systems, you need to use Hwloc v2.8.0 or newer, or you will experience undefined behavior. - This effect is not unique to the Hardware Locality library; this is why the Open MPI community recommends using as recent as possible versions of all support libraries. + * This library is required; Open MPI will not build without it. + * **Minimum version required:** |hwloc_min_version| + * **Version embedded in Open MPI distribution:** + |hwloc_embedded_version| .. danger:: As of |ompi_ver|, Open MPI does not yet support the Hwloc v3.x series (which may not even be available at @@ -77,6 +55,81 @@ process. More on this below. uses Hwloc, it uses the *same* Hwloc with which Open MPI was compiled. +* `Libevent `_ + + * This library is required; Open MPI will not build without it. + * **Minimum version required:** |event_min_version| + * **Version embedded in Open MPI distribution:** + |event_embedded_version| + +* `PMIx `_ + + * This library is required; Open MPI will not build without it. + * **Minimum version required when building without PRRTE:** + |pmix_min_version| + * **Minimum version required when building with PRRTE:** `See the + PRRTE project documentation `_. + * **Version embedded in Open MPI distribution:** + |pmix_embedded_version| + +* `PRRTE `_ + + * This library is optional in some environments. See below. + * **Minimum version required:** |prte_min_version| + + .. note:: While building Open MPI with PRRTE |prte_min_version| + *works*, you will not get a fully-populated + ``mpirun(1)`` man page. The Open MPI community + recommends that you use PRRTE version 3.0.1 or higher. + + * **Version embedded in Open MPI distribution:** + |prte_embedded_version| + + PRRTE provides Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI + application launchers (the two commands are identical; they are + symbolic links to the same executable). + + .. warning:: If you are building the PRRTE that is embedded in the + Open MPI |ompi_ver| distribution: + + * If you are also building the PMIx that is embedded in + the Open MPI |ompi_ver| distribution, that + combination of packages is supported. + + * If you are building against an external PMIx + installation (i.e., a version of PMIx that is not + embedded in the Open MPI |ompi_ver| distribution), + you should check `the PRRTE project documentation + `_ to see what minimum + version of PMIx is required. + + * If your environment uses another MPI application launcher (e.g., + Slurm users can use the ``srun`` launcher to "direct launch" Open + MPI applications), then the use of PRRTE is optional. + * If your environment has no other MPI application launcher, then + you need to install PRRTE and build Open MPI with PRRTE support. + * Open MPI can use the copy of PRRTE embedded in its source code + tree, or compile/link against an external PRRTE installation. + :ref:`See this section for details about how to specify each + method + `. + +.. note:: In general, the Open MPI community recommends using the most + recent versions of both the :ref:`required support libraries + ` and any other + optional support libraries. This is because more recent + versions typically tend to include bug fixes, sometimes + affecting Open MPI functionality. As a specific example, + there is a known issue with `Hardware Locality + `_ releases older + than v2.8.0 on systems with Intel Ponte Vecchio + accelerators. If you run Open MPI on such systems, you need + to use Hwloc v2.8.0 or newer, or you will experience + undefined behavior. This effect is not unique to the + Hardware Locality library; this is why the Open MPI + community recommends using as recent as possible versions of + *all* support libraries. + Library dependencies -------------------- @@ -145,6 +198,8 @@ example |mdash| only Libevent and Hwloc, that somewhat simplifies the final Open MPI configuration, and therefore avoids some potentially erroneous configurations. +.. _required-support-libraries-configure-discovery-label: + How ``configure`` finds the required libraries ---------------------------------------------- @@ -264,7 +319,7 @@ on Mac OS because: tarballs). #. In MacOS, it is common for `Homebrew `_ or `MacPorts `_ to install: - + * `Hardware Locality `_ * `Libevent `_